From ef619490f23877adbce37c9012496d01128bae4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 23:13:14 +0300 Subject: [PATCH] test(cluster): observe confirmed writes after longer rolling actions --- .github/pull_request_template.md | 31 +- Makefile | 4 +- REPORT.md | 43 + evidence/cluster-first-failed.json | 1839 +++++++++++++++++++++++++ evidence/cluster-first-failure.txt | 20 + evidence/gc-first.json | 209 +++ evidence/gc-seed-first.json | 20 + evidence/pipeline-versions-first.json | 980 +++++++++++++ research/analyze_pipeline.py | 4 +- scripts/verify_cluster.py | 7 +- 10 files changed, 3138 insertions(+), 19 deletions(-) create mode 100644 evidence/cluster-first-failed.json create mode 100644 evidence/cluster-first-failure.txt create mode 100644 evidence/gc-first.json create mode 100644 evidence/gc-seed-first.json create mode 100644 evidence/pipeline-versions-first.json diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a4..2a2dcabd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,19 @@ -## Лабораторная 5 +## Лабораторная5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Маслов, БВТ2301. Цель5, трек2: идемпотентность versioned image pipeline. +Собственная ЛР4 перенесена с объявленным переходом upload201→202queued. +Реализованы durable private S3 staging/reservation, atomic PG job+outbox, +persistent Valkey Streams/AOF/WAITAOF, relay/watchdog, отдельный bounded worker, +4variants, version/fence/live-lease CAS, finite retry/jitter/failed registry, +audited replay/newversion и safe retention cleanup. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Контракт и публичные tests сохранены. Запуск, схема, fault methodology, +результаты и ответы защиты — REPORT.md/SCHEMA.md.112HTTP и22unit прошли; +полный clean-clone repeat будет внесён перед сдачей. Промежуточные failed +harness runs сохранены и не выданы за полный PASS. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Ограничения: одинhost/SPOFs, нет production TLS/backup/global admission quota; +HEAD не periodic byte audit; at-least-once с идемпотентным эффектом, +не exactly-once delivery; удалённый CI не подтверждён. +При разработке кода, проверок и отчёта использована помощь ИИ. Источники — +teacher contracts, собственная ЛР4, pinned libraries и официальные Valkey/Pillow docs. diff --git a/Makefile b/Makefile index 6e04c7dd..1edbd7ad 100644 --- a/Makefile +++ b/Makefile @@ -39,8 +39,8 @@ up: bootstrap down: docker compose down lint: - $(UV) run --locked ruff format --check src student-tests student-unit scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) - $(UV) run --locked ruff check src student-tests student-unit scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) + $(UV) run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) + $(UV) run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) test-student: BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests verify: diff --git a/REPORT.md b/REPORT.md index 7c635047..8c0318b2 100644 --- a/REPORT.md +++ b/REPORT.md @@ -243,3 +243,46 @@ MAX_UPLOAD_BYTES/MAX_IMAGE_PIXELS и idempotency TTL документируют контракт реализации: s3:9000/us-east-1/messenger,10MiB/40M/24h. Их нельзя изменить одним редактированием env: соответствующие constants/SQL надо менять согласованно. Secret *_FILE paths заданы в compose; значения создаёт bootstrap. + +## Карта критериев и промежуточные результаты + +| Уровень / требование | Реализация / автоматическая проверка | +| --- | --- | +| 3: минимум ЛР4 +202queued | API/ACL/byte/pixel/MIME guards;112 HTTP tests, public smoke | +| 3: durable broker и отдельный worker | persistent Valkey Streams/AOF/WAITAOF; broker SIGKILL с volume | +| 3:4states, original/thumbnail, retry | src/worker.py, schema-valid metadata; failed/retry scenarios | +| 3:worker stop→queued link→restart→ready | commit_before_publish_and_worker_stopped | +| 4:crop/optimized/metadata removal | HTTP geometry/EXIF/alpha +22 unit tests | +| 4:outbox gap и publisher crash | API COMMIT-before-publish и relay XADD-before-PG-marker | +| 4:finite backoff/permanent/transient/replay |5attempt exhaustion, timeout, failed registry, audit, ready replay rejection | +| 4:crash после PUT доACK, lease, bounded slots | before_effect/after_effect/after_commit ×2; live state/actual limits проверяются в полном повторе | +| 4:cleanup после фиксации/retention | verify_cleanup seed/verify, реальныйS3возраст, dry/apply/repeat | +| 5:versioned idempotency track | duplicates/reorder и held stalev1→v2×2, no old publication/hash change | + +[evidence/http-first.json](evidence/http-first.json):112 HTTP PASS404.42s, +source/runtime bb4e8a2. Отдельные22unit PASS после добавления real pixel, +checksum и malformed image worker tests. Public smoke PASS. + +Первые полные matrix runs сохранены честно как незавершённые: +[pipeline-first-failed.json](evidence/pipeline-first-failed.json) —2cases, +затем тест сравнил очередь при работающем relay; snapshot изолирован остановкой +relay, повтор подтвердил сохранность exact entries после broker SIGKILL. +[pipeline-second-failed.json](evidence/pipeline-second-failed.json) —15cases +прошли, затем отдельные controller invocations съели12s бюджет старого child. +V2 была ready, но parent успел убитьv1; это ещё не доказательство CAS. + +Новая координация использует один заранее запущенный controller: ждёт phase, +создаётv2, ждёт ready, отпускаетv1 и видит stale_completion_rejected внутри +живого окна. [pipeline-versions-first.json](evidence/pipeline-versions-first.json) +— **scope=versions**,4cases PASS:2version races, malformed notification, +private/IAM/otherAPI. Это целевой повтор, а не19cases полного matrix. +Приложение в этих исправлениях harness не менялось; полный clean clone обязан +завершить scope=all, включая более строгие ACK/live-state/resource assertions. + +[evidence/gc-first.json](evidence/gc-first.json): реальный возраст orphan +1279.103s (выше300s); dry-run сохранил bytes, apply удалил expired objects, +повтор не изменил ничего. Active/current/fresh и неистёкший reservation +защищены; source_expired replay отвергнут; expired reservation больше не +проходит handoff predicate. Fixture metadata failed состарены SQL отдельно, +но возраст S3 LastModified не подменялся. Cleanup fixtures синтетические и +не обещают восстановление bytes из checksum. diff --git a/evidence/cluster-first-failed.json b/evidence/cluster-first-failed.json new file mode 100644 index 00000000..f13f4058 --- /dev/null +++ b/evidence/cluster-first-failed.json @@ -0,0 +1,1839 @@ +{ + "date": "2026-11-15T20:08:42.652330+00:00", + "source_commit": "972cef8f7f3938913beb56d32551a88f5e40bc88", + "complete": false, + "before": [ + { + "role": "api", + "container": "0c7b704276e3", + "pid": 75153, + "started_at": "2026-11-15T19:50:21.154879093Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "role": "api", + "container": "a90645b411d6", + "pid": 22120, + "started_at": "2026-11-15T20:06:45.480384299Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "role": "broker", + "container": "5d3ebf6f87ff", + "pid": 77503, + "started_at": "2026-11-15T19:51:03.049505001Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "6ad17613e620", + "pid": 60587, + "started_at": "2026-11-15T19:17:14.204239507Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "c6fb99c3ddbc", + "pid": 60326, + "started_at": "2026-11-15T19:45:43.876610632Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "c60ecc6c94f4", + "pid": 63147, + "started_at": "2026-11-15T19:17:36.952605129Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "76efa23ba942", + "pid": 60650, + "started_at": "2026-11-15T19:17:14.476492674Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "a990e3899324", + "pid": 27505, + "started_at": "2026-11-15T20:08:42.302428089Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "c8273b4552c9", + "pid": 98443, + "started_at": "2026-11-15T19:58:04.386027252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "de0c05e2e665", + "pid": 60843, + "started_at": "2026-11-15T19:17:15.006133424Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "a89e2cf493d9", + "pid": 18369, + "started_at": "2026-11-15T20:05:23.317701469Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ], + "parallel_replay": { + "requests": 30, + "messages": 1, + "instances": [ + "0024aba8755f4f3d92a85c4dbb751f5a", + "ee20d98fc7ac4ef08a41c2bc8fe82534" + ] + }, + "atomic_rollback_and_5xx_retry": true, + "expired_key_can_be_reused": true, + "failovers": [ + { + "confirmed_writes": 212, + "successful_reads": 200, + "errors": 13, + "successful_rps": 27.28, + "first_success_after_action_seconds": 5.028, + "action": { + "finished": 3.996 + }, + "observations": [ + { + "at": 0.008, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.067, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.124, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.205, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.265, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.321, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.392, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.46, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.517, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.576, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.639, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.701, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.759, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.816, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.871, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.936, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.996, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.063, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.134, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.192, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.25, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.312, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.37, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.426, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.495, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.557, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.623, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.685, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.745, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.809, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.871, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.933, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.035, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.138, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.236, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.337, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.456, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.519, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.577, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.64, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.701, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.795, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.895, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.995, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 3.101, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 3.196, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 3.252, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 3.322, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.388, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.445, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.506, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.563, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.616, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.667, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.72, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.778, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.831, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.888, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.95, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.028, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.092, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.149, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.209, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.269, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.329, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.389, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.454, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.515, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.581, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.641, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.703, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.76, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.817, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.881, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.94, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.005, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.073, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.138, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.201, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.268, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.33, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.392, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.45, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.512, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.582, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.639, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.697, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.757, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.847, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.944, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.049, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.147, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.246, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.348, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.443, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.503, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.563, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.624, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.686, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.748, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.81, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.872, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.933, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.0, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.063, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.134, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.196, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.264, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.33, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.401, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.464, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.532, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.604, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.669, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.732, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.795, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.865, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.928, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.99, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.078, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.146, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.208, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.27, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.334, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.4, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.465, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.527, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.594, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.661, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.724, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.783, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.844, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.903, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.968, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.032, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.1, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.159, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.22, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.289, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.355, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.424, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.493, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.56, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.634, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.714, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.774, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.838, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.905, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.973, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.035, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.1, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.172, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.233, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.294, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.354, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.415, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.479, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.541, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.603, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.662, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.728, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.791, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.849, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.907, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.966, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.03, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.103, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.169, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.236, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.303, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.372, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.436, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.501, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.566, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.643, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.712, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.78, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.847, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.915, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.981, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.049, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.119, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.188, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.249, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.31, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.37, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.437, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.499, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.559, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.62, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.682, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.742, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.801, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.858, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.915, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.973, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.033, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.093, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.152, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.216, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.277, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.345, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.41, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.475, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.538, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.601, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.667, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.737, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.812, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.876, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.941, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 15.008, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "0c7b704276e3", + "pid": 75153, + "started_at": "2026-11-15T19:50:21.154879093Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + } + }, + { + "confirmed_writes": 204, + "successful_reads": 191, + "errors": 14, + "successful_rps": 26.15, + "first_success_after_action_seconds": 4.344, + "action": { + "finished": 3.311 + }, + "observations": [ + { + "at": 0.021, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.091, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.157, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.216, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.276, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.343, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.433, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.554, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.628, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.717, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.784, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.864, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.94, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.011, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.106, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.178, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.237, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.341, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.436, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.54, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.647, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.748, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.843, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.904, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.978, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.082, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.179, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.282, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.381, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.484, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.543, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.606, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 2.667, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 2.725, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 2.787, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 2.844, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 2.899, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 2.953, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 3.011, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 3.063, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 3.113, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 3.165, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 3.216, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 3.268, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.344, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.409, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.477, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.552, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.628, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.687, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.744, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.798, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.856, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.918, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.983, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.045, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.106, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.182, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.262, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.329, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.389, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.453, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.521, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.59, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.651, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.713, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.825, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.891, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.975, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.034, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.106, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.171, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.232, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.288, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.347, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.411, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.481, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.545, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.611, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.686, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.758, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.817, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.878, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.943, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.007, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.064, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.119, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.18, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.263, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.329, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.386, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.442, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.501, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.558, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.617, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.685, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.755, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.82, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.885, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.945, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.007, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.072, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.139, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.202, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.269, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.333, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.392, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.451, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.515, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.574, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.644, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.709, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.775, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.842, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.9, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.96, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.034, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.093, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.156, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.219, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.3, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.491, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.594, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.693, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.798, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.887, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.991, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.089, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.152, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.213, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.271, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.333, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.397, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.454, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.522, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.591, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.658, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.728, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.797, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.864, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.924, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.986, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.041, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.101, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.16, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.217, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.276, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.356, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.432, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.492, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.556, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.621, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.691, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.755, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.82, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.894, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.956, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.016, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.076, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.137, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.198, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.258, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.327, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.392, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.46, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.539, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.604, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.669, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.737, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.81, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.873, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.938, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.003, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.061, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.122, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.184, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.25, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.307, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.367, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.44, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.506, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.568, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.634, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.697, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.765, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.831, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.903, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.978, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.04, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.101, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.16, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.222, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.285, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.355, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.415, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.472, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.537, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.6, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.664, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.73, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.797, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.862, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.93, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.998, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "a90645b411d6", + "pid": 22120, + "started_at": "2026-11-15T20:06:45.480384299Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + } + } + ] +} diff --git a/evidence/cluster-first-failure.txt b/evidence/cluster-first-failure.txt new file mode 100644 index 00000000..bb88731c --- /dev/null +++ b/evidence/cluster-first-failure.txt @@ -0,0 +1,20 @@ +Traceback (most recent call last): + File "", line 203, in _run_module_as_main + File "", line 88, in _run_code + File "/scripts/verify_cluster.py", line 342, in + main() + ~~~~^^ + File "/scripts/verify_cluster.py", line 278, in main + report["rolling"] = exercise( + ~~~~~~~~^ + client, + ^^^^^^^ + ...<3 lines>... + seconds=45, + ^^^^^^^^^^^ + ) + ^ + File "/scripts/verify_cluster.py", line 114, in exercise + assert recovery is not None and (seconds > 20 or recovery <= 10) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +AssertionError diff --git a/evidence/gc-first.json b/evidence/gc-first.json new file mode 100644 index 00000000..fed9a9e6 --- /dev/null +++ b/evidence/gc-first.json @@ -0,0 +1,209 @@ +{ + "source_commit": "972cef8f7f3938913beb56d32551a88f5e40bc88", + "seed": "gc-seed-first.json", + "complete": true, + "real_s3_age_seconds": 1279.102705, + "dry_run": { + "apply": false, + "retention_seconds": 300, + "actions": [ + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/crop", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/optimized", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/thumbnail", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/4d806c50-751a-4ea0-b4e0-06fd832a82ca/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/667404a0-d3d8-4d0e-b1b9-b362cea129a7/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/8f12140f-f650-415e-8881-e5707ed58127/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/original", + "applied": false + } + ] + }, + "apply": { + "apply": true, + "retention_seconds": 300, + "actions": [ + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/crop", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/optimized", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/thumbnail", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/4d806c50-751a-4ea0-b4e0-06fd832a82ca/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/667404a0-d3d8-4d0e-b1b9-b362cea129a7/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/8f12140f-f650-415e-8881-e5707ed58127/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/original", + "applied": true + } + ] + }, + "repeat": { + "apply": true, + "retention_seconds": 300, + "actions": [] + }, + "before": { + "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan": true, + "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original": true, + "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized": true, + "objects/staging/69150761-a8a1-437b-91b0-b363d7a1507a/original": true, + "objects/staging/ec7a6893-8d3b-4488-b0cd-06e53499cf9d/original": true, + "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/fresh-orphan": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/original": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/thumbnail": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/crop": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/optimized": true + }, + "after": { + "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan": false, + "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original": false, + "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original": false, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original": false, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail": false, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop": false, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized": false, + "objects/staging/69150761-a8a1-437b-91b0-b363d7a1507a/original": true, + "objects/staging/ec7a6893-8d3b-4488-b0cd-06e53499cf9d/original": true, + "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/fresh-orphan": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/original": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/thumbnail": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/crop": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/optimized": true + }, + "dry_run_unchanged": true, + "active_current_and_fresh_protected": true, + "expired_source_replay_refused": true, + "expired_writer_fenced": true +} diff --git a/evidence/gc-seed-first.json b/evidence/gc-seed-first.json new file mode 100644 index 00000000..229ed850 --- /dev/null +++ b/evidence/gc-seed-first.json @@ -0,0 +1,20 @@ +{ + "seed_id": "ae4b883f7b784f40ae8f5e1c78ac3091", + "source_commit": "a275cf2632d3bf3d96409408d9cb313cdb2824c2", + "date": "2026-11-15T19:46:23.318659+00:00", + "ready_attachment": "d6f10e70-74e9-4394-be76-5c0b14d3f91f", + "ready_job": "c0936959-f7ec-4fa2-a9ec-ee0de8d58d35", + "chat": "3428fad7-2178-4730-b56a-4b579bdd4966", + "owner": "36007eaa-ceba-4f80-98fa-13c912f73873", + "version2_job": "cc18c75e-8886-4333-bae8-5da93d626c73", + "orphan": "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan", + "failed_id": "b98bc768-f36a-4615-9097-c93a78bfed05", + "active_id": "69150761-a8a1-437b-91b0-b363d7a1507a", + "reservation_id": "ec7a6893-8d3b-4488-b0cd-06e53499cf9d", + "expired_reservation_id": "402b3515-91ad-40e0-8e36-6627feb18be7", + "failed_source": "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original", + "active_source": "objects/staging/69150761-a8a1-437b-91b0-b363d7a1507a/original", + "reservation_key": "objects/staging/ec7a6893-8d3b-4488-b0cd-06e53499cf9d/original", + "expired_reservation_key": "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original", + "seeded_unix": 1794771995.922627 +} diff --git a/evidence/pipeline-versions-first.json b/evidence/pipeline-versions-first.json new file mode 100644 index 00000000..abecfe1f --- /dev/null +++ b/evidence/pipeline-versions-first.json @@ -0,0 +1,980 @@ +{ + "source_commit": "972cef8f7f3938913beb56d32551a88f5e40bc88", + "date": "2026-11-15T20:03:53.705513+00:00", + "track": 2, + "scope": "versions", + "complete": true, + "cases": [ + { + "case": "stale_v1_after_v2_repeat_1", + "v1": { + "job": { + "id": "51c2387d-8bc4-4f37-9ad9-b722464c1f46", + "attachment_id": "4567437f-228f-4cd6-be3b-0609664645cd", + "pipeline_version": 1, + "source_key": "objects/staging/4567437f-228f-4cd6-be3b-0609664645cd/original", + "state": "obsolete", + "attempt": 1, + "cycle": 1, + "fence": 2, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:03:59.131799+00:00", + "error_code": null, + "created_at": "2026-11-15 20:03:59.131799+00:00", + "updated_at": "2026-11-15 20:04:25.401001+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/crop", + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/optimized", + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/original", + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794773039190-0", + "outbox": 43 + }, + "created_at": "2026-11-15 20:03:59.200524+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794773054245-0", + "outbox": 44 + }, + "created_at": "2026-11-15 20:04:14.251008+00:00" + }, + { + "event": "version_controller_waiting", + "detail": {}, + "created_at": "2026-11-15 20:04:18.270502+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:04:25.403107+00:00" + }, + { + "event": "stale_completion_rejected", + "detail": { + "fence": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:04:30.862701+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773039190-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:30.980982+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773054245-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:30.994675+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "stale" + }, + "created_at": "2026-11-15 20:04:31.083095+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079900-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.913386+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079902-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.914038+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079904-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.920903+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079905-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.922028+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079906-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.925707+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079908-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.926905+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079909-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.930906+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079910-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.931649+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079911-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.935325+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079912-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.935997+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079913-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.940184+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079914-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.940743+00:00" + } + ] + }, + "v2": { + "job": { + "id": "49ab086a-9b0e-4185-a4a7-1c95c546aed9", + "attachment_id": "4567437f-228f-4cd6-be3b-0609664645cd", + "pipeline_version": 2, + "source_key": "objects/staging/4567437f-228f-4cd6-be3b-0609664645cd/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:04:28.320656+00:00", + "error_code": null, + "created_at": "2026-11-15 20:04:28.320656+00:00", + "updated_at": "2026-11-15 20:04:30.796918+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/optimized", + "sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/crop", + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/optimized", + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/original", + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/thumbnail" + ], + "events": [ + { + "event": "new_version", + "detail": { + "previous_job": "51c2387d-8bc4-4f37-9ad9-b722464c1f46" + }, + "created_at": "2026-11-15 20:04:28.323557+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794773068363-0", + "outbox": 45 + }, + "created_at": "2026-11-15 20:04:28.367667+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 2 + }, + "created_at": "2026-11-15 20:04:28.373092+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 2, + "variants": 4 + }, + "created_at": "2026-11-15 20:04:30.798523+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.29709320799975103, + "cpu_seconds": 0.24198554100000003, + "rss_peak_mib": 79.61328125 + }, + "created_at": "2026-11-15 20:04:30.965510+00:00" + } + ] + }, + "new_hashes_unchanged": true + }, + { + "case": "stale_v1_after_v2_repeat_2", + "v1": { + "job": { + "id": "8679296c-7a76-46b4-a6a4-02b57d0dedaf", + "attachment_id": "0a837770-bc8b-40c0-b155-a1d29956e589", + "pipeline_version": 1, + "source_key": "objects/staging/0a837770-bc8b-40c0-b155-a1d29956e589/original", + "state": "obsolete", + "attempt": 1, + "cycle": 1, + "fence": 2, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:05:03.917398+00:00", + "error_code": null, + "created_at": "2026-11-15 20:05:03.917399+00:00", + "updated_at": "2026-11-15 20:05:25.442589+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/crop", + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/optimized", + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/original", + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794773103958-0", + "outbox": 46 + }, + "created_at": "2026-11-15 20:05:03.964545+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794773119171-0", + "outbox": 47 + }, + "created_at": "2026-11-15 20:05:19.176271+00:00" + }, + { + "event": "version_controller_waiting", + "detail": {}, + "created_at": "2026-11-15 20:05:22.667065+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:05:25.444410+00:00" + }, + { + "event": "stale_completion_rejected", + "detail": { + "fence": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:05:30.563412+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773103958-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:30.764786+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773119171-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:30.775876+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "stale" + }, + "created_at": "2026-11-15 20:05:30.815598+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139795-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.811610+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139792-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.811671+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139797-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.818733+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139798-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.819185+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139799-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.824008+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139800-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.824598+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139801-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.829008+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139803-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.829676+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139804-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.833617+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139805-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.834470+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139805-1", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.838465+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139806-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.839569+00:00" + } + ] + }, + "v2": { + "job": { + "id": "3c673acd-d8c5-44c7-8366-eed0480acdd8", + "attachment_id": "0a837770-bc8b-40c0-b155-a1d29956e589", + "pipeline_version": 2, + "source_key": "objects/staging/0a837770-bc8b-40c0-b155-a1d29956e589/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:05:28.099769+00:00", + "error_code": null, + "created_at": "2026-11-15 20:05:28.099769+00:00", + "updated_at": "2026-11-15 20:05:30.544288+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/optimized", + "sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/crop", + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/optimized", + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/original", + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/thumbnail" + ], + "events": [ + { + "event": "new_version", + "detail": { + "previous_job": "8679296c-7a76-46b4-a6a4-02b57d0dedaf" + }, + "created_at": "2026-11-15 20:05:28.102749+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794773128278-0", + "outbox": 48 + }, + "created_at": "2026-11-15 20:05:28.288692+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 2 + }, + "created_at": "2026-11-15 20:05:28.307421+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 2, + "variants": 4 + }, + "created_at": "2026-11-15 20:05:30.544711+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.28706112500003655, + "cpu_seconds": 0.24108328799999978, + "rss_peak_mib": 79.671875 + }, + "created_at": "2026-11-15 20:05:30.752666+00:00" + } + ] + }, + "new_hashes_unchanged": true + }, + { + "case": "malformed_notification_rejected", + "durable_diagnostic": true + }, + { + "case": "private_iam_and_other_api_download", + "security": { + "anonymous": 403, + "iam": { + "create_bucket": "AccessDenied", + "write_outside_prefix": "AccessDenied", + "list_other_bucket": "AccessDenied" + } + }, + "attachment": "5875dfe8-09d9-4ded-9e31-a9329f0974ac", + "uploader": "c12dfe7a14a648809e07d709180d70e3", + "downloader": "0024aba8755f4f3d92a85c4dbb751f5a", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df" + } + ], + "worker_metrics": [ + { + "job_id": "947fae60-fb2b-4000-ac82-76c0aa946f17", + "detail": { + "result": "ready", + "seconds": 0.5540742509999745, + "cpu_seconds": 0.47041774699999994, + "rss_peak_mib": 79.33203125 + } + }, + { + "job_id": "ac292276-9fc0-4d20-b9d0-1c784017f555", + "detail": { + "result": "ready", + "seconds": 0.26326079199998276, + "cpu_seconds": 0.23446212599999994, + "rss_peak_mib": 79.6796875 + } + }, + { + "job_id": "4d1ee9b7-32e0-44c4-8264-a831b9bc3bdb", + "detail": { + "result": "ready", + "seconds": 0.2615961250003238, + "cpu_seconds": 0.22782362500000009, + "rss_peak_mib": 79.44140625 + } + }, + { + "job_id": "1174230c-fb4f-4cdc-9db8-896d002f90a4", + "detail": { + "result": "ready", + "seconds": 0.2803810010000234, + "cpu_seconds": 0.24079599299999987, + "rss_peak_mib": 79.34765625 + } + }, + { + "job_id": "4b28f16f-162d-4f60-b1fd-3a907ee834e1", + "detail": { + "result": "ready", + "seconds": 0.2567913339998995, + "cpu_seconds": 0.228462414, + "rss_peak_mib": 79.47265625 + } + }, + { + "job_id": "12fa59f7-ed8a-4810-95cf-8b9be4c020e8", + "detail": { + "result": "ready", + "seconds": 0.28690104199995403, + "cpu_seconds": 0.23826428900000018, + "rss_peak_mib": 79.3046875 + } + }, + { + "job_id": "6247813c-df74-4eeb-9326-cbb13f860336", + "detail": { + "result": "ready", + "seconds": 0.4403786249999939, + "cpu_seconds": 0.4114635, + "rss_peak_mib": 108.4140625 + } + }, + { + "job_id": "9a3fad2f-b3cc-4a28-b996-10c84a4c4b94", + "detail": { + "result": "ready", + "seconds": 0.2648974580001777, + "cpu_seconds": 0.23192129000000028, + "rss_peak_mib": 79.3515625 + } + }, + { + "job_id": "c0936959-f7ec-4fa2-a9ec-ee0de8d58d35", + "detail": { + "result": "ready", + "seconds": 0.30638329200019143, + "cpu_seconds": 0.27590770600000014, + "rss_peak_mib": 79.4296875 + } + }, + { + "job_id": "cc18c75e-8886-4333-bae8-5da93d626c73", + "detail": { + "result": "ready", + "seconds": 0.2798381249999693, + "cpu_seconds": 0.24821178899999996, + "rss_peak_mib": 79.6640625 + } + }, + { + "job_id": "1016e737-7ec1-4fde-9946-a342d70b2a71", + "detail": { + "result": "ready", + "seconds": 0.2740797910000765, + "cpu_seconds": 0.2354738329999997, + "rss_peak_mib": 79.3125 + } + }, + { + "job_id": "6e004e09-d91d-42b9-907b-9fda074afeb3", + "detail": { + "result": "ready", + "seconds": 0.2604502920003142, + "cpu_seconds": 0.22423492300000003, + "rss_peak_mib": 79.328125 + } + }, + { + "job_id": "817ac0d3-2ae3-4721-a1c4-0d517f624f3a", + "detail": { + "result": "ready", + "seconds": 0.2753011250001691, + "cpu_seconds": 0.24031520900000003, + "rss_peak_mib": 79.31640625 + } + }, + { + "job_id": "a68327bb-1d99-4ca1-a541-c197ad0683ea", + "detail": { + "result": "ready", + "seconds": 0.2575967509997099, + "cpu_seconds": 0.23045399499999997, + "rss_peak_mib": 79.84765625 + } + }, + { + "job_id": "2b40b034-2ea5-4612-8d4e-f099e1e3f65d", + "detail": { + "result": "ready", + "seconds": 0.2825338749998991, + "cpu_seconds": 0.24960387800000006, + "rss_peak_mib": 79.30859375 + } + }, + { + "job_id": "681235f5-558b-44b8-af12-e61d2f6c65bf", + "detail": { + "result": "ready", + "seconds": 0.2647618330001933, + "cpu_seconds": 0.23189429199999978, + "rss_peak_mib": 79.6796875 + } + }, + { + "job_id": "e4b36262-0688-4b3c-90ce-9252e9633353", + "detail": { + "result": "ready", + "seconds": 0.2914882079999188, + "cpu_seconds": 0.24316999600000022, + "rss_peak_mib": 79.80078125 + } + }, + { + "job_id": "e1ebdd4b-996f-41d2-aa54-246cc9f4c8d3", + "detail": { + "result": "ready", + "seconds": 0.259311875000094, + "cpu_seconds": 0.23085762499999984, + "rss_peak_mib": 79.45703125 + } + }, + { + "job_id": "fbedc7f8-97a6-4c59-848c-d4ab8045d4d2", + "detail": { + "result": "ready", + "seconds": 0.26234775000011723, + "cpu_seconds": 0.22806161800000013, + "rss_peak_mib": 79.328125 + } + }, + { + "job_id": "be6bd110-e0e9-4dbf-92cb-2ee8e18d407d", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "be6bd110-e0e9-4dbf-92cb-2ee8e18d407d", + "detail": { + "result": "ready", + "seconds": 0.30820225099978416, + "cpu_seconds": 0.27197879400000025, + "rss_peak_mib": 79.34765625 + } + }, + { + "job_id": "5d2588f5-72b5-41ce-95ce-c78327ceddeb", + "detail": { + "code": "worker_timeout", + "result": "transient" + } + }, + { + "job_id": "5d2588f5-72b5-41ce-95ce-c78327ceddeb", + "detail": { + "result": "ready", + "seconds": 0.26163899999983187, + "cpu_seconds": 0.22942266700000014, + "rss_peak_mib": 79.33203125 + } + }, + { + "job_id": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1", + "detail": { + "code": "invalid_image", + "result": "permanent" + } + }, + { + "job_id": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1", + "detail": { + "result": "ready", + "seconds": 0.27349454200020773, + "cpu_seconds": 0.23941921199999983, + "rss_peak_mib": 79.28125 + } + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "detail": { + "result": "ready", + "seconds": 0.2960601250001673, + "cpu_seconds": 0.26317221, + "rss_peak_mib": 79.25 + } + }, + { + "job_id": "20c34eb3-0582-4144-847c-fa037647f863", + "detail": { + "result": "ready", + "seconds": 0.3045327089998864, + "cpu_seconds": 0.2630334579999998, + "rss_peak_mib": 79.578125 + } + }, + { + "job_id": "c013ea06-4dea-4e65-b715-8a3f5976fa57", + "detail": { + "result": "ready", + "seconds": 0.3056550410001364, + "cpu_seconds": 0.26324616899999986, + "rss_peak_mib": 79.65234375 + } + }, + { + "job_id": "44de0dbe-f2b8-40bd-8b65-479e8f7243b5", + "detail": { + "code": "worker_timeout", + "result": "transient" + } + }, + { + "job_id": "fe00834b-4f3b-4d9b-bf47-c9cec70caac9", + "detail": { + "result": "ready", + "seconds": 0.26621149999982663, + "cpu_seconds": 0.23299866699999994, + "rss_peak_mib": 79.67578125 + } + }, + { + "job_id": "49ab086a-9b0e-4185-a4a7-1c95c546aed9", + "detail": { + "result": "ready", + "seconds": 0.29709320799975103, + "cpu_seconds": 0.24198554100000003, + "rss_peak_mib": 79.61328125 + } + }, + { + "job_id": "51c2387d-8bc4-4f37-9ad9-b722464c1f46", + "detail": { + "result": "stale" + } + }, + { + "job_id": "3c673acd-d8c5-44c7-8366-eed0480acdd8", + "detail": { + "result": "ready", + "seconds": 0.28706112500003655, + "cpu_seconds": 0.24108328799999978, + "rss_peak_mib": 79.671875 + } + }, + { + "job_id": "8679296c-7a76-46b4-a6a4-02b57d0dedaf", + "detail": { + "result": "stale" + } + }, + { + "job_id": "8eedc435-5509-4724-9a18-e478d920f101", + "detail": { + "result": "ready", + "seconds": 0.32150650000039604, + "cpu_seconds": 0.28338829200000015, + "rss_peak_mib": 79.37109375 + } + } + ], + "audit": [ + { + "job_id": "cc18c75e-8886-4333-bae8-5da93d626c73", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 19:46:27.434253+00:00" + }, + { + "job_id": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1", + "action": "replay", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 19:56:33.964020+00:00" + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "action": "replay", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 19:57:37.397099+00:00" + }, + { + "job_id": "fe00834b-4f3b-4d9b-bf47-c9cec70caac9", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 19:59:07.104714+00:00" + }, + { + "job_id": "49ab086a-9b0e-4185-a4a7-1c95c546aed9", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Held stale writer race", + "created_at": "2026-11-15 20:04:28.322925+00:00" + }, + { + "job_id": "3c673acd-d8c5-44c7-8366-eed0480acdd8", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Held stale writer race", + "created_at": "2026-11-15 20:05:28.102142+00:00" + } + ] +} diff --git a/research/analyze_pipeline.py b/research/analyze_pipeline.py index 2436dc16..7a9e0d84 100644 --- a/research/analyze_pipeline.py +++ b/research/analyze_pipeline.py @@ -59,10 +59,12 @@ def main(): ) stale = [r for r in rows if r["state"] == "obsolete"] assert all(r["publications"] == 0 for r in stale) + job_ids = {r["job_id"] for r in rows} samples = [ x["detail"] for x in d["worker_metrics"] - if x["detail"].get("result") == "ready" + if str(x["job_id"]) in job_ids + and x["detail"].get("result") == "ready" and "rss_peak_mib" in x["detail"] ] stats = { diff --git a/scripts/verify_cluster.py b/scripts/verify_cluster.py index a0dbd639..19be4197 100644 --- a/scripts/verify_cluster.py +++ b/scripts/verify_cluster.py @@ -59,7 +59,12 @@ def exercise(client, session, chat, action, *, seconds=15): thread.start() key = uuid4().hex try: - while time.monotonic() - started < seconds or not finished.is_set(): + while ( + time.monotonic() - started < seconds + or not finished.is_set() + or time.monotonic() - started + < action_result.get("finished", 0) + 2 + ): h = {**auth(session), "Idempotency-Key": key} try: r = client.post(path, headers=h, json={"text": key})