From 442163b0f0a549ef065cb13ee39b5ca2f8580c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 23:47:51 +0300 Subject: [PATCH] fix(pipeline): reclaim pending entries after the live lease budget --- REPORT.md | 16 + evidence/clean-clone-first.json | 8796 +++++++++++++++++++++++++++++++ scripts/verify_pipeline.py | 22 +- src/worker.py | 6 +- 4 files changed, 8835 insertions(+), 5 deletions(-) create mode 100644 evidence/clean-clone-first.json diff --git a/REPORT.md b/REPORT.md index 83821142..d2d9c12e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -304,3 +304,19 @@ API failovers (writes/errors/recovery seconds): 214/6/3.086, 216/8/2.762. Rolling: 730 confirmed writes, 18 transient errors; все исходныеAPI заменены, данные/replay сохранены. Public failover/persistence PASS; StartedAt PG/cache/S3/broker/LB изменились, прежний token и file сохранились. Это не zero-error rolling. + +## Уточнение строгого окна COMMIT→ACK + +Первый чистый повтор [clean-clone-first.json](evidence/clean-clone-first.json) +sourceeb6d229 прошёл112HTTP379.34s/22unit/19cases/cluster/cleanup/hashes. +Однако в after_commit snapshot уже был duplicate_decision: второйconsumer +XAUTOCLAIM после3s успевал ACK раньшеSIGKILL. Это безопасно для данных, +но не доказывает конкретное окно COMMIT-before-ACK. Первый receipt сохранён +как результат прежней версии проверок; строгая граница требует нового повтора. + +Reclaim idle теперь равен lease(23s normal/15s test), превышая child timeout +20s/12s. Это предотвращает ранний захват чужого live entry. Новый тест +снимком XPENDING+XRANGE подтверждает конкретные job entries доSIGKILL и +отсутствие child_finished; после restart требует ready duplicate decision +и исчезновение notifications. Scope=all, данные и все прежние assertions +сохранены. Финальным доказательством станет clean-clone-final.json. diff --git a/evidence/clean-clone-first.json b/evidence/clean-clone-first.json new file mode 100644 index 00000000..535b6246 --- /dev/null +++ b/evidence/clean-clone-first.json @@ -0,0 +1,8796 @@ +{ + "date": "2026-11-15T20:20:18.627769+00:00", + "source_commit": "eb6d229afb1437cd0f8b5e2ee0ca384d584ddd8c", + "fresh_secrets_environment_and_volumes": true, + "command": "python scripts/verify_clean.py", + "fixed_host_port_for_restart": 61281, + "static_checks": ".venv/bin/python scripts/check_contract.py\nPASS: offline skeleton structure, local refs and Python syntax; application NOT tested.\n.venv/bin/python scripts/validate_openapi.py\nPASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.\n/.local/bin/uv run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/verify_clean.py scripts/verify_cleanup.py scripts/verify_cluster.py scripts/verify_pipeline.py\n38 files already formatted\n/.local/bin/uv run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/verify_clean.py scripts/verify_cleanup.py scripts/verify_cluster.py scripts/verify_pipeline.py\nAll checks passed!", + "unit_tests": "PYTHONPATH=. /.local/bin/uv run --locked pytest student-unit -q\n...................... [100%]\n22 passed in 14.11s", + "smoke": "python3 tests/smoke.py --base-url \"http://127.0.0.1:61281\" \nPASS: lab5 public API scenario; infrastructure and grading evidence remain separate.", + "http_tests": "........................................................................ [ 64%]\n........................................ [100%]\n112 passed in 379.34s (0:06:19)", + "pipeline": { + "source_commit": "eb6d229afb1437cd0f8b5e2ee0ca384d584ddd8c", + "date": "2026-11-15T20:31:54.110235+00:00", + "track": 2, + "scope": "all", + "complete": true, + "cases": [ + { + "case": "staging_before_handoff_crash", + "no_202": true, + "no_attachment_or_job_created": true, + "orphan_retained_for_cleanup": true + }, + { + "case": "commit_before_publish_and_worker_stopped", + "attachment": "b93c8166-bdaa-4645-aa38-f9565856d888", + "effect": { + "job": { + "id": "7e6614be-b097-46d0-8398-47ba5ce3444a", + "attachment_id": "b93c8166-bdaa-4645-aa38-f9565856d888", + "pipeline_version": 1, + "source_key": "objects/staging/b93c8166-bdaa-4645-aa38-f9565856d888/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:32:28.373080+00:00", + "error_code": null, + "created_at": "2026-11-15 20:32:28.373080+00:00", + "updated_at": "2026-11-15 20:32:54.127104+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/crop", + "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/optimized", + "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/original", + "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:32:51.920425+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794774771894-0", + "outbox": 13 + }, + "created_at": "2026-11-15 20:32:51.963453+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:32:53.891667+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:32:54.127296+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2639559589997589, + "cpu_seconds": 0.23261253799999992, + "rss_peak_mib": 79.73046875 + }, + "created_at": "2026-11-15 20:32:54.259195+00:00" + } + ] + }, + "queued_link_allowed": true, + "api_and_broker_restarted": true, + "broker_stopped_during_upload": true + }, + { + "case": "confirmed_broker_entry_survives_sigkill", + "effect": { + "job": { + "id": "c7761c1b-92ce-40cd-9058-bdad8f5f189c", + "attachment_id": "6ad9c0b0-4b93-4129-b066-270b361e9256", + "pipeline_version": 1, + "source_key": "objects/staging/6ad9c0b0-4b93-4129-b066-270b361e9256/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:33:00.555147+00:00", + "error_code": null, + "created_at": "2026-11-15 20:33:00.555148+00:00", + "updated_at": "2026-11-15 20:33:28.308832+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/crop", + "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/optimized", + "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/original", + "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774780583-0", + "outbox": 14 + }, + "created_at": "2026-11-15 20:33:00.588385+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:33:26.112048+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:33:28.043378+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:33:28.309061+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2951308749998134, + "cpu_seconds": 0.2648634940000001, + "rss_peak_mib": 79.609375 + }, + "created_at": "2026-11-15 20:33:28.419122+00:00" + } + ] + }, + "same_stream_entries": true, + "entry_count": 1 + }, + { + "case": "relay_crash_after_xadd_before_pg_marker", + "unpublished_after_crash": 1, + "effect": { + "job": { + "id": "0048f677-7bca-4f87-8cdf-1a60d6e3bd2c", + "attachment_id": "c660e20a-a0cd-402e-9062-8e4aa3631ef7", + "pipeline_version": 1, + "source_key": "objects/staging/c660e20a-a0cd-402e-9062-8e4aa3631ef7/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:33:35.232787+00:00", + "error_code": null, + "created_at": "2026-11-15 20:33:35.232787+00:00", + "updated_at": "2026-11-15 20:34:04.233613+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/crop", + "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/optimized", + "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/original", + "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:34:02.035998+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:34:03.981197+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:34:04.233799+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2813682500000141, + "cpu_seconds": 0.23825850199999987, + "rss_peak_mib": 79.734375 + }, + "created_at": "2026-11-15 20:34:04.343185+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794774845269-0", + "outbox": 15 + }, + "created_at": "2026-11-15 20:34:05.343360+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794774845269-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:34:05.346880+00:00" + } + ] + } + }, + { + "case": "before_effect_sigkill_repeat_1", + "at_crash": { + "job": { + "id": "6cc8e812-e4ae-4d36-a230-d5235fa45b4b", + "attachment_id": "420109c5-01d0-42de-82ba-06d85d0df610", + "pipeline_version": 1, + "source_key": "objects/staging/420109c5-01d0-42de-82ba-06d85d0df610/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "f7bc3cf0-c12c-49aa-bb37-ec0f0fb4937d-1", + "lease_until": "2026-11-15 20:34:36.244033+00:00", + "next_at": "2026-11-15 20:34:10.365420+00:00", + "error_code": null, + "created_at": "2026-11-15 20:34:10.365420+00:00", + "updated_at": "2026-11-15 20:34:21.244033+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774850500-0", + "outbox": 16 + }, + "created_at": "2026-11-15 20:34:10.505316+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:34:21.245618+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:34:23.191221+00:00" + } + ] + }, + "effect": { + "job": { + "id": "6cc8e812-e4ae-4d36-a230-d5235fa45b4b", + "attachment_id": "420109c5-01d0-42de-82ba-06d85d0df610", + "pipeline_version": 1, + "source_key": "objects/staging/420109c5-01d0-42de-82ba-06d85d0df610/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:34:37.016501+00:00", + "error_code": null, + "created_at": "2026-11-15 20:34:10.365420+00:00", + "updated_at": "2026-11-15 20:34:39.412593+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/crop", + "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/optimized", + "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/original", + "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774850500-0", + "outbox": 16 + }, + "created_at": "2026-11-15 20:34:10.505316+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:34:21.245618+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:34:23.191221+00:00" + }, + { + "event": "retry", + "detail": { + "code": "lease_expired", + "attempt": 1, + "backoff_seconds": 0.7349758269229827 + }, + "created_at": "2026-11-15 20:34:36.286227+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794774877116-0", + "outbox": 17 + }, + "created_at": "2026-11-15 20:34:37.122203+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:34:37.129858+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794774850500-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:34:37.408734+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:34:39.170954+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:34:39.412973+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.27222879200053285, + "cpu_seconds": 0.2380743289999998, + "rss_peak_mib": 79.97265625 + }, + "created_at": "2026-11-15 20:34:39.527386+00:00" + } + ] + } + }, + { + "case": "after_effect_sigkill_repeat_1", + "at_crash": { + "job": { + "id": "beeb7929-64e8-4ed5-9990-16f0dd1d5aa8", + "attachment_id": "83cc6e0a-1397-4939-a0af-5527f3975b56", + "pipeline_version": 1, + "source_key": "objects/staging/83cc6e0a-1397-4939-a0af-5527f3975b56/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "455238d9-616c-4663-8081-b75644c34445-1", + "lease_until": "2026-11-15 20:35:12.872510+00:00", + "next_at": "2026-11-15 20:34:45.852662+00:00", + "error_code": null, + "created_at": "2026-11-15 20:34:45.852662+00:00", + "updated_at": "2026-11-15 20:34:57.872510+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/crop", + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/optimized", + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/original", + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774885985-0", + "outbox": 18 + }, + "created_at": "2026-11-15 20:34:45.991777+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:34:57.875075+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:35:00.481122+00:00" + } + ] + }, + "effect": { + "job": { + "id": "beeb7929-64e8-4ed5-9990-16f0dd1d5aa8", + "attachment_id": "83cc6e0a-1397-4939-a0af-5527f3975b56", + "pipeline_version": 1, + "source_key": "objects/staging/83cc6e0a-1397-4939-a0af-5527f3975b56/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:35:11.577038+00:00", + "error_code": null, + "created_at": "2026-11-15 20:34:45.852662+00:00", + "updated_at": "2026-11-15 20:35:14.926277+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/crop", + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/optimized", + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/original", + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774885985-0", + "outbox": 18 + }, + "created_at": "2026-11-15 20:34:45.991777+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:34:57.875075+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:35:00.481122+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "worker_timeout", + "result": "transient" + }, + "created_at": "2026-11-15 20:35:09.925522+00:00" + }, + { + "event": "retry", + "detail": { + "code": "worker_timeout", + "attempt": 1, + "backoff_seconds": 1.650536463275796 + }, + "created_at": "2026-11-15 20:35:09.927630+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794774911683-0", + "outbox": 19 + }, + "created_at": "2026-11-15 20:35:11.687860+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:35:12.689706+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:35:14.686531+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:35:14.926555+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2685703340002874, + "cpu_seconds": 0.23196675100000008, + "rss_peak_mib": 79.7109375 + }, + "created_at": "2026-11-15 20:35:15.204431+00:00" + } + ] + } + }, + { + "case": "after_commit_sigkill_repeat_1", + "at_crash": { + "job": { + "id": "08f2419e-cb14-41d5-b2b2-5e0e36b023a6", + "attachment_id": "42e402e7-8c37-4c43-a0cd-b456575b21b6", + "pipeline_version": 1, + "source_key": "objects/staging/42e402e7-8c37-4c43-a0cd-b456575b21b6/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:35:21.265720+00:00", + "error_code": null, + "created_at": "2026-11-15 20:35:21.265720+00:00", + "updated_at": "2026-11-15 20:35:34.120048+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/crop", + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/optimized", + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/original", + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774921333-0", + "outbox": 20 + }, + "created_at": "2026-11-15 20:35:21.338179+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:35:31.994723+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:35:33.882444+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:35:34.120287+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794774921333-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:35:35.241237+00:00" + } + ] + }, + "effect": { + "job": { + "id": "08f2419e-cb14-41d5-b2b2-5e0e36b023a6", + "attachment_id": "42e402e7-8c37-4c43-a0cd-b456575b21b6", + "pipeline_version": 1, + "source_key": "objects/staging/42e402e7-8c37-4c43-a0cd-b456575b21b6/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:35:21.265720+00:00", + "error_code": null, + "created_at": "2026-11-15 20:35:21.265720+00:00", + "updated_at": "2026-11-15 20:35:34.120048+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/crop", + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/optimized", + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/original", + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774921333-0", + "outbox": 20 + }, + "created_at": "2026-11-15 20:35:21.338179+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:35:31.994723+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:35:33.882444+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:35:34.120287+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794774921333-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:35:35.241237+00:00" + } + ] + } + }, + { + "case": "before_effect_sigkill_repeat_2", + "at_crash": { + "job": { + "id": "347bd84b-610b-427f-98e0-b6a290923456", + "attachment_id": "aeb85ccc-0787-49a8-910d-54a4c083316d", + "pipeline_version": 1, + "source_key": "objects/staging/aeb85ccc-0787-49a8-910d-54a4c083316d/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "402beccc-189b-41d8-9bcf-641b86b96677-0", + "lease_until": "2026-11-15 20:36:29.671877+00:00", + "next_at": "2026-11-15 20:36:03.390772+00:00", + "error_code": null, + "created_at": "2026-11-15 20:36:03.390772+00:00", + "updated_at": "2026-11-15 20:36:14.671878+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774963487-0", + "outbox": 21 + }, + "created_at": "2026-11-15 20:36:03.493220+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:36:14.673658+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:36:16.642679+00:00" + } + ] + }, + "effect": { + "job": { + "id": "347bd84b-610b-427f-98e0-b6a290923456", + "attachment_id": "aeb85ccc-0787-49a8-910d-54a4c083316d", + "pipeline_version": 1, + "source_key": "objects/staging/aeb85ccc-0787-49a8-910d-54a4c083316d/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:36:31.201421+00:00", + "error_code": null, + "created_at": "2026-11-15 20:36:03.390772+00:00", + "updated_at": "2026-11-15 20:36:33.481230+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/crop", + "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/optimized", + "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/original", + "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774963487-0", + "outbox": 21 + }, + "created_at": "2026-11-15 20:36:03.493220+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:36:14.673658+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:36:16.642679+00:00" + }, + { + "event": "retry", + "detail": { + "code": "lease_expired", + "attempt": 1, + "backoff_seconds": 1.32552866661181 + }, + "created_at": "2026-11-15 20:36:29.877852+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794774963487-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:36:30.532350+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794774991334-0", + "outbox": 22 + }, + "created_at": "2026-11-15 20:36:31.340026+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:36:31.343727+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:36:33.246295+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:36:33.481412+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2644206660006603, + "cpu_seconds": 0.23357183000000004, + "rss_peak_mib": 79.86328125 + }, + "created_at": "2026-11-15 20:36:33.606213+00:00" + } + ] + } + }, + { + "case": "after_effect_sigkill_repeat_2", + "at_crash": { + "job": { + "id": "ea0a2fc7-4484-46fa-8c13-b4523eebd673", + "attachment_id": "590de76f-eca1-4cac-8623-81d980778c96", + "pipeline_version": 1, + "source_key": "objects/staging/590de76f-eca1-4cac-8623-81d980778c96/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "b37ae4d8-b8e2-414f-9051-5a71c065b53d-0", + "lease_until": "2026-11-15 20:37:05.257675+00:00", + "next_at": "2026-11-15 20:36:39.256083+00:00", + "error_code": null, + "created_at": "2026-11-15 20:36:39.256083+00:00", + "updated_at": "2026-11-15 20:36:50.257676+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/crop", + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/optimized", + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/original", + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774999364-0", + "outbox": 23 + }, + "created_at": "2026-11-15 20:36:39.425553+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:36:50.267695+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:36:52.972919+00:00" + } + ] + }, + "effect": { + "job": { + "id": "ea0a2fc7-4484-46fa-8c13-b4523eebd673", + "attachment_id": "590de76f-eca1-4cac-8623-81d980778c96", + "pipeline_version": 1, + "source_key": "objects/staging/590de76f-eca1-4cac-8623-81d980778c96/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:37:03.206829+00:00", + "error_code": null, + "created_at": "2026-11-15 20:36:39.256083+00:00", + "updated_at": "2026-11-15 20:37:07.376667+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/crop", + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/optimized", + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/original", + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774999364-0", + "outbox": 23 + }, + "created_at": "2026-11-15 20:36:39.425553+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:36:50.267695+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:36:52.972919+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "worker_timeout", + "result": "transient" + }, + "created_at": "2026-11-15 20:37:02.332522+00:00" + }, + { + "event": "retry", + "detail": { + "code": "worker_timeout", + "attempt": 1, + "backoff_seconds": 0.8732405781066988 + }, + "created_at": "2026-11-15 20:37:02.337632+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775023321-0", + "outbox": 24 + }, + "created_at": "2026-11-15 20:37:03.327334+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:37:05.140639+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:37:07.142690+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:37:07.376943+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2620389579997209, + "cpu_seconds": 0.22754557900000028, + "rss_peak_mib": 79.73046875 + }, + "created_at": "2026-11-15 20:37:07.477626+00:00" + } + ] + } + }, + { + "case": "after_commit_sigkill_repeat_2", + "at_crash": { + "job": { + "id": "760d739e-0946-468a-a5a9-12b2894d6c27", + "attachment_id": "30baebd5-f3c3-44f7-91b5-57fe4503861c", + "pipeline_version": 1, + "source_key": "objects/staging/30baebd5-f3c3-44f7-91b5-57fe4503861c/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:37:13.467787+00:00", + "error_code": null, + "created_at": "2026-11-15 20:37:13.467787+00:00", + "updated_at": "2026-11-15 20:37:27.022215+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/crop", + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/optimized", + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/original", + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775033601-0", + "outbox": 25 + }, + "created_at": "2026-11-15 20:37:13.607336+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:37:24.631354+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:37:26.772981+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:37:27.022580+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775033601-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:37:27.861083+00:00" + } + ] + }, + "effect": { + "job": { + "id": "760d739e-0946-468a-a5a9-12b2894d6c27", + "attachment_id": "30baebd5-f3c3-44f7-91b5-57fe4503861c", + "pipeline_version": 1, + "source_key": "objects/staging/30baebd5-f3c3-44f7-91b5-57fe4503861c/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:37:13.467787+00:00", + "error_code": null, + "created_at": "2026-11-15 20:37:13.467787+00:00", + "updated_at": "2026-11-15 20:37:27.022215+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/crop", + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/optimized", + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/original", + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775033601-0", + "outbox": 25 + }, + "created_at": "2026-11-15 20:37:13.607336+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:37:24.631354+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:37:26.772981+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:37:27.022580+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775033601-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:37:27.861083+00:00" + } + ] + } + }, + { + "case": "transient_retry_with_jitter", + "effect": { + "job": { + "id": "bce86e16-6c52-4947-8b94-ed17b69df55b", + "attachment_id": "07f472a7-9b00-4153-8307-b06bd0a39a8d", + "pipeline_version": 1, + "source_key": "objects/staging/07f472a7-9b00-4153-8307-b06bd0a39a8d/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:38:09.399222+00:00", + "error_code": null, + "created_at": "2026-11-15 20:37:55.139843+00:00", + "updated_at": "2026-11-15 20:38:11.677100+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/crop", + "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/optimized", + "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/original", + "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775075318-0", + "outbox": 26 + }, + "created_at": "2026-11-15 20:37:55.323741+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:38:06.421657+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:38:08.356592+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 20:38:08.456980+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 1, + "backoff_seconds": 0.941461107889648 + }, + "created_at": "2026-11-15 20:38:08.458651+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775089510-0", + "outbox": 27 + }, + "created_at": "2026-11-15 20:38:09.515268+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:38:09.518861+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:38:11.430068+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:38:11.677376+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.28160174999993615, + "cpu_seconds": 0.24880712500000013, + "rss_peak_mib": 79.8515625 + }, + "created_at": "2026-11-15 20:38:11.832807+00:00" + } + ] + } + }, + { + "case": "hung_child_killed_and_retried", + "effect": { + "job": { + "id": "0c0fa874-bbb1-4e6e-8c4d-d2123250f72e", + "attachment_id": "06b673c6-800b-4688-8ea8-4e42e396dc93", + "pipeline_version": 1, + "source_key": "objects/staging/06b673c6-800b-4688-8ea8-4e42e396dc93/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:38:41.110764+00:00", + "error_code": null, + "created_at": "2026-11-15 20:38:17.573851+00:00", + "updated_at": "2026-11-15 20:38:43.355872+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/crop", + "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/optimized", + "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/original", + "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775097743-0", + "outbox": 28 + }, + "created_at": "2026-11-15 20:38:17.757424+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:38:28.429239+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:38:30.339378+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "worker_timeout", + "result": "transient" + }, + "created_at": "2026-11-15 20:38:40.544843+00:00" + }, + { + "event": "retry", + "detail": { + "code": "worker_timeout", + "attempt": 1, + "backoff_seconds": 0.5645840246252455 + }, + "created_at": "2026-11-15 20:38:40.547793+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775121151-0", + "outbox": 29 + }, + "created_at": "2026-11-15 20:38:41.159193+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:38:41.164202+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:38:43.119014+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:38:43.356116+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.26547958299943275, + "cpu_seconds": 0.23297874699999999, + "rss_peak_mib": 79.734375 + }, + "created_at": "2026-11-15 20:38:43.455687+00:00" + } + ] + }, + "observed_limits": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + } + }, + { + "case": "permanent_failed_registry_and_audited_replay", + "failed": { + "id": "4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8", + "chat_id": "98d8f122-f8cf-435c-be59-583b97667d1e", + "owner_id": "fa26fdd2-4267-4706-964c-ebd760e343a1", + "filename": "synthetic.png", + "content_type": "image/png", + "size_bytes": 742, + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "created_at": "2026-11-15T20:38:48.980408Z", + "updated_at": "2026-11-15T20:39:03.438755Z", + "status": "failed", + "variants": [], + "error_code": "invalid_image" + }, + "effect": { + "job": { + "id": "441e0f5a-cf4e-4d34-b08d-8bf63a5ace71", + "attachment_id": "4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8", + "pipeline_version": 1, + "source_key": "objects/staging/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/original", + "state": "ready", + "attempt": 1, + "cycle": 2, + "fence": 4, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:39:07.659023+00:00", + "error_code": null, + "created_at": "2026-11-15 20:38:48.982040+00:00", + "updated_at": "2026-11-15 20:39:09.979880+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/crop", + "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/optimized", + "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/original", + "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775129172-0", + "outbox": 30 + }, + "created_at": "2026-11-15 20:38:49.176239+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:39:00.596129+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:02.599916+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "invalid_image", + "result": "permanent" + }, + "created_at": "2026-11-15 20:39:03.430340+00:00" + }, + { + "event": "failed", + "detail": { + "code": "invalid_image", + "attempt": 1, + "backoff_seconds": 0.7867716366060661 + }, + "created_at": "2026-11-15 20:39:03.439770+00:00" + }, + { + "event": "replay", + "detail": { + "previous_job": "441e0f5a-cf4e-4d34-b08d-8bf63a5ace71" + }, + "created_at": "2026-11-15 20:39:07.661357+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775147735-0", + "outbox": 31 + }, + "created_at": "2026-11-15 20:39:07.740726+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 4, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:39:07.750907+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:09.734619+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 4, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:39:09.980287+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.28228149999995367, + "cpu_seconds": 0.24383666199999987, + "rss_peak_mib": 79.75 + }, + "created_at": "2026-11-15 20:39:10.113396+00:00" + } + ] + }, + "ready_replay_rejected": true + }, + { + "case": "finite_retry_exhaustion_then_explicit_replay", + "failed": { + "id": "6e40e091-61d8-4b8f-b4dc-2e26dbe1b019", + "chat_id": "98d8f122-f8cf-435c-be59-583b97667d1e", + "owner_id": "fa26fdd2-4267-4706-964c-ebd760e343a1", + "filename": "synthetic.png", + "content_type": "image/png", + "size_bytes": 742, + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "created_at": "2026-11-15T20:39:20.743932Z", + "updated_at": "2026-11-15T20:39:50.591547Z", + "status": "failed", + "variants": [], + "error_code": "retry_exhausted" + }, + "effect": { + "job": { + "id": "030165d4-972d-4456-a8d9-955a85120a09", + "attachment_id": "6e40e091-61d8-4b8f-b4dc-2e26dbe1b019", + "pipeline_version": 1, + "source_key": "objects/staging/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/original", + "state": "ready", + "attempt": 1, + "cycle": 2, + "fence": 12, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:39:59.841431+00:00", + "error_code": null, + "created_at": "2026-11-15 20:39:20.745669+00:00", + "updated_at": "2026-11-15 20:40:02.325527+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/crop", + "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/optimized", + "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/original", + "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775160881-0", + "outbox": 32 + }, + "created_at": "2026-11-15 20:39:20.885407+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:39:32.235402+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:34.125168+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 20:39:34.211760+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 1, + "backoff_seconds": 1.2866648375283887 + }, + "created_at": "2026-11-15 20:39:34.212727+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775175536-0", + "outbox": 33 + }, + "created_at": "2026-11-15 20:39:35.541877+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:39:35.545139+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:37.402676+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 20:39:37.495151+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 2, + "backoff_seconds": 1.0989196801088283 + }, + "created_at": "2026-11-15 20:39:37.496314+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775178620-0", + "outbox": 34 + }, + "created_at": "2026-11-15 20:39:38.628009+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 5, + "attempt": 3, + "version": 1 + }, + "created_at": "2026-11-15 20:39:38.630658+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:40.657197+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 20:39:40.765673+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 3, + "backoff_seconds": 2.912731852995912 + }, + "created_at": "2026-11-15 20:39:40.766978+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775183797-0", + "outbox": 35 + }, + "created_at": "2026-11-15 20:39:43.801337+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 7, + "attempt": 4, + "version": 1 + }, + "created_at": "2026-11-15 20:39:43.802540+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:45.671882+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 20:39:45.781450+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 4, + "backoff_seconds": 2.594216138460152 + }, + "created_at": "2026-11-15 20:39:45.782611+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775188560-0", + "outbox": 36 + }, + "created_at": "2026-11-15 20:39:48.563912+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 9, + "attempt": 5, + "version": 1 + }, + "created_at": "2026-11-15 20:39:48.566032+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:50.487885+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 20:39:50.590927+00:00" + }, + { + "event": "failed", + "detail": { + "code": "retry_exhausted", + "attempt": 5, + "backoff_seconds": 21.98370831486453 + }, + "created_at": "2026-11-15 20:39:50.592015+00:00" + }, + { + "event": "replay", + "detail": { + "previous_job": "030165d4-972d-4456-a8d9-955a85120a09" + }, + "created_at": "2026-11-15 20:39:59.843878+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775200050-0", + "outbox": 37 + }, + "created_at": "2026-11-15 20:40:00.056086+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 12, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:40:00.064203+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:40:02.013527+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 12, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:40:02.325858+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.3446279999998296, + "cpu_seconds": 0.3051448689999998, + "rss_peak_mib": 79.73046875 + }, + "created_at": "2026-11-15 20:40:02.476674+00:00" + } + ] + } + }, + { + "case": "reordered_and_duplicate_notifications", + "effects": [ + { + "job": { + "id": "a3878536-8835-4ff8-b051-3af704a36b8f", + "attachment_id": "aae81ab4-ced7-4b50-a3e2-1bd38ed5a234", + "pipeline_version": 1, + "source_key": "objects/staging/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:40:10.338027+00:00", + "error_code": null, + "created_at": "2026-11-15 20:40:10.338027+00:00", + "updated_at": "2026-11-15 20:40:32.822839+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/crop", + "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/optimized", + "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/original", + "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:40:30.495056+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:40:32.568451+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:40:32.823097+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.28610912500062113, + "cpu_seconds": 0.2469917960000001, + "rss_peak_mib": 79.7890625 + }, + "created_at": "2026-11-15 20:40:32.991116+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227777-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.078142+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227778-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.078392+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227779-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.090334+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227781-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.090559+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227781-1", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.103713+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227820-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.104109+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227822-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.119854+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775234392-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:34.400641+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775234392-0", + "outbox": 38 + }, + "created_at": "2026-11-15 20:40:34.468800+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244937-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:44.976495+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244934-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:44.976628+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244939-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:44.990009+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244940-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:44.990346+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244940-1", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:44.999297+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244941-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:45.000627+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244942-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:45.008765+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244943-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:45.010144+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244944-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:45.022585+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244945-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:45.022915+00:00" + } + ] + }, + { + "job": { + "id": "06e45874-a726-4e60-9b00-2fa39f91462a", + "attachment_id": "c11f5af4-44f2-47df-93d9-9248fe267bec", + "pipeline_version": 1, + "source_key": "objects/staging/c11f5af4-44f2-47df-93d9-9248fe267bec/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:40:15.202254+00:00", + "error_code": null, + "created_at": "2026-11-15 20:40:15.202254+00:00", + "updated_at": "2026-11-15 20:40:32.822963+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/crop", + "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/optimized", + "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/original", + "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:40:30.446987+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:40:32.567586+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:40:32.823310+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.28661216700038494, + "cpu_seconds": 0.24859508499999983, + "rss_peak_mib": 79.33203125 + }, + "created_at": "2026-11-15 20:40:32.991322+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223439-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.695830+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223440-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.696231+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223442-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.705671+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223441-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.705689+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223443-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.714961+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223444-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.715387+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223444-1", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.726372+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775234472-0", + "outbox": 39 + }, + "created_at": "2026-11-15 20:40:34.474328+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775234472-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:34.479232+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249028-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.043428+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249025-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.043807+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249029-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.054945+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249031-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.056420+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249032-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.067328+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249033-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.067641+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249034-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.077663+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249035-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.077980+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249036-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.087385+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249038-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.087714+00:00" + } + ] + } + ], + "added_duplicates": 36, + "no_extra_attempt_after_ready": true + }, + { + "case": "stale_v1_after_v2_repeat_1", + "v1": { + "job": { + "id": "90ce943c-1a07-4c16-80d4-8d8f25423f6e", + "attachment_id": "d93fddc0-ac9a-40bf-8f43-45f56f976d25", + "pipeline_version": 1, + "source_key": "objects/staging/d93fddc0-ac9a-40bf-8f43-45f56f976d25/original", + "state": "obsolete", + "attempt": 1, + "cycle": 1, + "fence": 2, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:41:08.452978+00:00", + "error_code": null, + "created_at": "2026-11-15 20:41:08.452978+00:00", + "updated_at": "2026-11-15 20:41:34.395893+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v1/crop", + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v1/optimized", + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v1/original", + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775268516-0", + "outbox": 40 + }, + "created_at": "2026-11-15 20:41:08.526228+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775283705-0", + "outbox": 41 + }, + "created_at": "2026-11-15 20:41:23.709433+00:00" + }, + { + "event": "version_controller_waiting", + "detail": {}, + "created_at": "2026-11-15 20:41:27.122131+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:41:34.397674+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:41:36.282725+00:00" + }, + { + "event": "stale_completion_rejected", + "detail": { + "fence": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:41:38.834601+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775268516-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:38.887075+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775283705-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:38.892943+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "stale" + }, + "created_at": "2026-11-15 20:41:38.992209+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308761-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.770748+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308764-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.771391+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308765-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.781251+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308766-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.782276+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308768-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.789264+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308769-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.789785+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308769-1", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.794573+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308770-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.795175+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308771-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.799185+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308773-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.799791+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308774-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.804072+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308774-1", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.804819+00:00" + } + ] + }, + "v2": { + "job": { + "id": "a131fd82-344a-4a6a-9b20-06e70b896d0b", + "attachment_id": "d93fddc0-ac9a-40bf-8f43-45f56f976d25", + "pipeline_version": 2, + "source_key": "objects/staging/d93fddc0-ac9a-40bf-8f43-45f56f976d25/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:41:36.591313+00:00", + "error_code": null, + "created_at": "2026-11-15 20:41:36.591313+00:00", + "updated_at": "2026-11-15 20:41:38.753737+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/optimized", + "sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/crop", + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/optimized", + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/original", + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/thumbnail" + ], + "events": [ + { + "event": "new_version", + "detail": { + "previous_job": "90ce943c-1a07-4c16-80d4-8d8f25423f6e" + }, + "created_at": "2026-11-15 20:41:36.593847+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775296653-0", + "outbox": 42 + }, + "created_at": "2026-11-15 20:41:36.658428+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 2 + }, + "created_at": "2026-11-15 20:41:36.659959+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:41:38.510633+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 2, + "variants": 4 + }, + "created_at": "2026-11-15 20:41:38.754008+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.27137612600017746, + "cpu_seconds": 0.24279383700000023, + "rss_peak_mib": 79.78125 + }, + "created_at": "2026-11-15 20:41:38.874944+00:00" + } + ] + }, + "new_hashes_unchanged": true + }, + { + "case": "stale_v1_after_v2_repeat_2", + "v1": { + "job": { + "id": "c9ff05d2-7fe8-4b1f-934a-e83572707614", + "attachment_id": "4a818886-a36d-492f-a356-8f119eb36457", + "pipeline_version": 1, + "source_key": "objects/staging/4a818886-a36d-492f-a356-8f119eb36457/original", + "state": "obsolete", + "attempt": 1, + "cycle": 1, + "fence": 2, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:42:12.041598+00:00", + "error_code": null, + "created_at": "2026-11-15 20:42:12.041598+00:00", + "updated_at": "2026-11-15 20:42:37.014637+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/4a818886-a36d-492f-a356-8f119eb36457/v1/crop", + "objects/4a818886-a36d-492f-a356-8f119eb36457/v1/optimized", + "objects/4a818886-a36d-492f-a356-8f119eb36457/v1/original", + "objects/4a818886-a36d-492f-a356-8f119eb36457/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775332158-0", + "outbox": 43 + }, + "created_at": "2026-11-15 20:42:12.165123+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775347185-0", + "outbox": 44 + }, + "created_at": "2026-11-15 20:42:27.194548+00:00" + }, + { + "event": "version_controller_waiting", + "detail": {}, + "created_at": "2026-11-15 20:42:30.145665+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:42:37.016243+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:42:39.040876+00:00" + }, + { + "event": "stale_completion_rejected", + "detail": { + "fence": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:42:41.648675+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775332158-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:41.820730+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775347185-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:41.836455+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "stale" + }, + "created_at": "2026-11-15 20:42:41.907292+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370021-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.030566+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370018-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.030569+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370023-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.041619+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370025-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.042919+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370026-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.048086+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370028-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.048296+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370029-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.053246+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370030-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.053925+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370031-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.059020+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370033-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.059407+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370035-1", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.065127+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370035-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.065845+00:00" + } + ] + }, + "v2": { + "job": { + "id": "d3a419fa-e9ca-4ca8-9e62-4fe224ef6ccf", + "attachment_id": "4a818886-a36d-492f-a356-8f119eb36457", + "pipeline_version": 2, + "source_key": "objects/staging/4a818886-a36d-492f-a356-8f119eb36457/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:42:39.348499+00:00", + "error_code": null, + "created_at": "2026-11-15 20:42:39.348499+00:00", + "updated_at": "2026-11-15 20:42:41.599185+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/optimized", + "sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/crop", + "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/optimized", + "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/original", + "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/thumbnail" + ], + "events": [ + { + "event": "new_version", + "detail": { + "previous_job": "c9ff05d2-7fe8-4b1f-934a-e83572707614" + }, + "created_at": "2026-11-15 20:42:39.351053+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775359398-0", + "outbox": 45 + }, + "created_at": "2026-11-15 20:42:39.403387+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 2 + }, + "created_at": "2026-11-15 20:42:39.412390+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:42:41.344884+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 2, + "variants": 4 + }, + "created_at": "2026-11-15 20:42:41.599440+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.28745358400010446, + "cpu_seconds": 0.2529595779999998, + "rss_peak_mib": 79.85546875 + }, + "created_at": "2026-11-15 20:42:41.808234+00:00" + } + ] + }, + "new_hashes_unchanged": true + }, + { + "case": "malformed_notification_rejected", + "durable_diagnostic": true + }, + { + "case": "private_iam_and_other_api_download", + "security": { + "anonymous": 403, + "iam": { + "create_bucket": "AccessDenied", + "write_outside_prefix": "AccessDenied", + "list_other_bucket": "AccessDenied" + } + }, + "attachment": "5a62a72b-e2b0-424f-926b-9e4a22c9f53d", + "uploader": "ea289153015c4784a656d9b1d302dbde", + "downloader": "5f8f83d988fc4479b4865a203c766f1f", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df" + } + ], + "worker_metrics": [ + { + "job_id": "5b31c9b5-e415-4916-b879-f863098efb3f", + "detail": { + "result": "ready", + "seconds": 0.2819997910000893, + "cpu_seconds": 0.24735475399999984, + "rss_peak_mib": 79.609375 + } + }, + { + "job_id": "ebfdafb4-422c-4dab-84fc-8cac7b6690a5", + "detail": { + "result": "ready", + "seconds": 0.2749179169995841, + "cpu_seconds": 0.24024937599999996, + "rss_peak_mib": 79.61328125 + } + }, + { + "job_id": "fb8e6292-7c89-4985-9379-d6129d94b8bc", + "detail": { + "result": "ready", + "seconds": 0.2542630829993868, + "cpu_seconds": 0.22826758300000005, + "rss_peak_mib": 79.72265625 + } + }, + { + "job_id": "082b41be-2f81-4127-9340-bc9276fa01b1", + "detail": { + "result": "ready", + "seconds": 0.2787759169996207, + "cpu_seconds": 0.24001637300000023, + "rss_peak_mib": 79.48828125 + } + }, + { + "job_id": "52e9a948-1efd-4914-95f9-5d3a3fc7d2fc", + "detail": { + "result": "ready", + "seconds": 0.29629000000022643, + "cpu_seconds": 0.2464472950000003, + "rss_peak_mib": 79.72265625 + } + }, + { + "job_id": "f60bb029-9f97-485f-a301-ef3d7ad509c4", + "detail": { + "result": "ready", + "seconds": 0.2661684159993456, + "cpu_seconds": 0.23351224500000023, + "rss_peak_mib": 79.859375 + } + }, + { + "job_id": "60afcdf4-d40f-4c90-999f-7c668eb2390f", + "detail": { + "result": "ready", + "seconds": 0.485136500000408, + "cpu_seconds": 0.4508282039999998, + "rss_peak_mib": 108.73828125 + } + }, + { + "job_id": "246bb511-bdc1-4a45-a0ee-fd20ab22f2f9", + "detail": { + "result": "ready", + "seconds": 0.2561037919995215, + "cpu_seconds": 0.2209544109999999, + "rss_peak_mib": 79.50390625 + } + }, + { + "job_id": "89c40d3f-711a-430c-a175-6a1a0c48ef9e", + "detail": { + "result": "ready", + "seconds": 0.3143088329998136, + "cpu_seconds": 0.27227425500000013, + "rss_peak_mib": 79.75 + } + }, + { + "job_id": "dad2632b-589e-4fa6-9f7f-d416444590b9", + "detail": { + "result": "ready", + "seconds": 0.2876035839999531, + "cpu_seconds": 0.24817545600000024, + "rss_peak_mib": 79.7265625 + } + }, + { + "job_id": "5c269448-69fd-47bc-8eae-e066be853c8e", + "detail": { + "result": "ready", + "seconds": 0.29481404200032557, + "cpu_seconds": 0.263315376, + "rss_peak_mib": 79.734375 + } + }, + { + "job_id": "fe1c5591-95fc-4ba3-88fb-2fa26badcbde", + "detail": { + "result": "ready", + "seconds": 0.31842683299964847, + "cpu_seconds": 0.26803228499999987, + "rss_peak_mib": 79.734375 + } + }, + { + "job_id": "7e6614be-b097-46d0-8398-47ba5ce3444a", + "detail": { + "result": "ready", + "seconds": 0.2639559589997589, + "cpu_seconds": 0.23261253799999992, + "rss_peak_mib": 79.73046875 + } + }, + { + "job_id": "c7761c1b-92ce-40cd-9058-bdad8f5f189c", + "detail": { + "result": "ready", + "seconds": 0.2951308749998134, + "cpu_seconds": 0.2648634940000001, + "rss_peak_mib": 79.609375 + } + }, + { + "job_id": "0048f677-7bca-4f87-8cdf-1a60d6e3bd2c", + "detail": { + "result": "ready", + "seconds": 0.2813682500000141, + "cpu_seconds": 0.23825850199999987, + "rss_peak_mib": 79.734375 + } + }, + { + "job_id": "6cc8e812-e4ae-4d36-a230-d5235fa45b4b", + "detail": { + "result": "ready", + "seconds": 0.27222879200053285, + "cpu_seconds": 0.2380743289999998, + "rss_peak_mib": 79.97265625 + } + }, + { + "job_id": "beeb7929-64e8-4ed5-9990-16f0dd1d5aa8", + "detail": { + "code": "worker_timeout", + "result": "transient" + } + }, + { + "job_id": "beeb7929-64e8-4ed5-9990-16f0dd1d5aa8", + "detail": { + "result": "ready", + "seconds": 0.2685703340002874, + "cpu_seconds": 0.23196675100000008, + "rss_peak_mib": 79.7109375 + } + }, + { + "job_id": "347bd84b-610b-427f-98e0-b6a290923456", + "detail": { + "result": "ready", + "seconds": 0.2644206660006603, + "cpu_seconds": 0.23357183000000004, + "rss_peak_mib": 79.86328125 + } + }, + { + "job_id": "ea0a2fc7-4484-46fa-8c13-b4523eebd673", + "detail": { + "code": "worker_timeout", + "result": "transient" + } + }, + { + "job_id": "ea0a2fc7-4484-46fa-8c13-b4523eebd673", + "detail": { + "result": "ready", + "seconds": 0.2620389579997209, + "cpu_seconds": 0.22754557900000028, + "rss_peak_mib": 79.73046875 + } + }, + { + "job_id": "bce86e16-6c52-4947-8b94-ed17b69df55b", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "bce86e16-6c52-4947-8b94-ed17b69df55b", + "detail": { + "result": "ready", + "seconds": 0.28160174999993615, + "cpu_seconds": 0.24880712500000013, + "rss_peak_mib": 79.8515625 + } + }, + { + "job_id": "0c0fa874-bbb1-4e6e-8c4d-d2123250f72e", + "detail": { + "code": "worker_timeout", + "result": "transient" + } + }, + { + "job_id": "0c0fa874-bbb1-4e6e-8c4d-d2123250f72e", + "detail": { + "result": "ready", + "seconds": 0.26547958299943275, + "cpu_seconds": 0.23297874699999999, + "rss_peak_mib": 79.734375 + } + }, + { + "job_id": "441e0f5a-cf4e-4d34-b08d-8bf63a5ace71", + "detail": { + "code": "invalid_image", + "result": "permanent" + } + }, + { + "job_id": "441e0f5a-cf4e-4d34-b08d-8bf63a5ace71", + "detail": { + "result": "ready", + "seconds": 0.28228149999995367, + "cpu_seconds": 0.24383666199999987, + "rss_peak_mib": 79.75 + } + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "detail": { + "result": "ready", + "seconds": 0.3446279999998296, + "cpu_seconds": 0.3051448689999998, + "rss_peak_mib": 79.73046875 + } + }, + { + "job_id": "a3878536-8835-4ff8-b051-3af704a36b8f", + "detail": { + "result": "ready", + "seconds": 0.28610912500062113, + "cpu_seconds": 0.2469917960000001, + "rss_peak_mib": 79.7890625 + } + }, + { + "job_id": "06e45874-a726-4e60-9b00-2fa39f91462a", + "detail": { + "result": "ready", + "seconds": 0.28661216700038494, + "cpu_seconds": 0.24859508499999983, + "rss_peak_mib": 79.33203125 + } + }, + { + "job_id": "a131fd82-344a-4a6a-9b20-06e70b896d0b", + "detail": { + "result": "ready", + "seconds": 0.27137612600017746, + "cpu_seconds": 0.24279383700000023, + "rss_peak_mib": 79.78125 + } + }, + { + "job_id": "90ce943c-1a07-4c16-80d4-8d8f25423f6e", + "detail": { + "result": "stale" + } + }, + { + "job_id": "d3a419fa-e9ca-4ca8-9e62-4fe224ef6ccf", + "detail": { + "result": "ready", + "seconds": 0.28745358400010446, + "cpu_seconds": 0.2529595779999998, + "rss_peak_mib": 79.85546875 + } + }, + { + "job_id": "c9ff05d2-7fe8-4b1f-934a-e83572707614", + "detail": { + "result": "stale" + } + }, + { + "job_id": "dda2169e-2083-40ef-8a3e-0c385a3f03d4", + "detail": { + "result": "ready", + "seconds": 0.30348837499968795, + "cpu_seconds": 0.2656605380000001, + "rss_peak_mib": 79.73828125 + } + } + ], + "audit": [ + { + "job_id": "fe1c5591-95fc-4ba3-88fb-2fa26badcbde", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 20:31:45.094363+00:00" + }, + { + "job_id": "441e0f5a-cf4e-4d34-b08d-8bf63a5ace71", + "action": "replay", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 20:39:07.660718+00:00" + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "action": "replay", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 20:39:59.843247+00:00" + }, + { + "job_id": "a131fd82-344a-4a6a-9b20-06e70b896d0b", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Held stale writer race", + "created_at": "2026-11-15 20:41:36.593403+00:00" + }, + { + "job_id": "d3a419fa-e9ca-4ca8-9e62-4fe224ef6ccf", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Held stale writer race", + "created_at": "2026-11-15 20:42:39.350444+00:00" + } + ] + }, + "cleanup": { + "source_commit": "eb6d229afb1437cd0f8b5e2ee0ca384d584ddd8c", + "seed": "clean-gc-seed.json", + "complete": true, + "real_s3_age_seconds": 733.85265, + "dry_run": { + "apply": false, + "retention_seconds": 300, + "actions": [ + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/crop", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/optimized", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/thumbnail", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/cleanup-3174841402a74a1fb5b6e7241a5e8d2a/old-orphan", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/30baebd5-f3c3-44f7-91b5-57fe4503861c/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/42e402e7-8c37-4c43-a0cd-b456575b21b6/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/6df2b381-f693-4cc3-be87-ceced86eecb8/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/ef82bc3b-0c79-482f-b73f-156b7f378015/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/f973cbdb-f407-4a48-914e-fdf85f1a258d/original", + "applied": false + } + ] + }, + "apply": { + "apply": true, + "retention_seconds": 300, + "actions": [ + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/crop", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/optimized", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/thumbnail", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/cleanup-3174841402a74a1fb5b6e7241a5e8d2a/old-orphan", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/30baebd5-f3c3-44f7-91b5-57fe4503861c/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/42e402e7-8c37-4c43-a0cd-b456575b21b6/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/6df2b381-f693-4cc3-be87-ceced86eecb8/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/ef82bc3b-0c79-482f-b73f-156b7f378015/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/f973cbdb-f407-4a48-914e-fdf85f1a258d/original", + "applied": true + } + ] + }, + "repeat": { + "apply": true, + "retention_seconds": 300, + "actions": [] + }, + "before": { + "objects/cleanup-3174841402a74a1fb5b6e7241a5e8d2a/old-orphan": true, + "objects/staging/6df2b381-f693-4cc3-be87-ceced86eecb8/original": true, + "objects/staging/f973cbdb-f407-4a48-914e-fdf85f1a258d/original": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/original": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/thumbnail": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/crop": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/optimized": true, + "objects/staging/d86d3319-0b87-443b-a8e7-b9d9e803c2e0/original": true, + "objects/staging/6ca87961-0147-4da6-a1d5-6207025ecde6/original": true, + "objects/cleanup-3174841402a74a1fb5b6e7241a5e8d2a/fresh-orphan": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/original": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/thumbnail": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/crop": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/optimized": true + }, + "after": { + "objects/cleanup-3174841402a74a1fb5b6e7241a5e8d2a/old-orphan": false, + "objects/staging/6df2b381-f693-4cc3-be87-ceced86eecb8/original": false, + "objects/staging/f973cbdb-f407-4a48-914e-fdf85f1a258d/original": false, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/original": false, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/thumbnail": false, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/crop": false, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/optimized": false, + "objects/staging/d86d3319-0b87-443b-a8e7-b9d9e803c2e0/original": true, + "objects/staging/6ca87961-0147-4da6-a1d5-6207025ecde6/original": true, + "objects/cleanup-3174841402a74a1fb5b6e7241a5e8d2a/fresh-orphan": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/original": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/thumbnail": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/crop": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/optimized": true + }, + "dry_run_unchanged": true, + "active_current_and_fresh_protected": true, + "expired_source_replay_refused": true, + "expired_writer_fenced": true + }, + "cluster": { + "date": "2026-11-15T20:28:04.394923+00:00", + "source_commit": "eb6d229afb1437cd0f8b5e2ee0ca384d584ddd8c", + "complete": true, + "before": [ + { + "role": "api", + "container": "8bbd861211fd", + "pid": 59296, + "started_at": "2026-11-15T20:21:21.104054635Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "079d9eff1d0c4c8cbc0fc0d4b89cb455" + }, + { + "role": "api", + "container": "ebc7e4c3838d", + "pid": 59353, + "started_at": "2026-11-15T20:21:21.27865926Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "role": "broker", + "container": "55ab9f986bae", + "pid": 58562, + "started_at": "2026-11-15T20:21:12.353588672Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "7014391ecf89", + "pid": 58493, + "started_at": "2026-11-15T20:21:12.04480938Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "fa3c50f83a1f", + "pid": 58431, + "started_at": "2026-11-15T20:21:11.739337255Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "242a49094f0d", + "pid": 60206, + "started_at": "2026-11-15T20:21:33.497731001Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "61281" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "932bb6635829", + "pid": 58277, + "started_at": "2026-11-15T20:21:11.12347638Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "6f6af6bdcd00", + "pid": 59239, + "started_at": "2026-11-15T20:21:20.927591635Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "a4dba713c111", + "pid": 58990, + "started_at": "2026-11-15T20:21:17.651360383Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "ca00d9061bb8", + "pid": 58361, + "started_at": "2026-11-15T20:21:11.435409755Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "1963232f6596", + "pid": 59409, + "started_at": "2026-11-15T20:21:21.488607385Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ], + "parallel_replay": { + "requests": 30, + "messages": 1, + "instances": [ + "079d9eff1d0c4c8cbc0fc0d4b89cb455", + "6e400225cb214074bf1ffd479b9a828e" + ] + }, + "atomic_rollback_and_5xx_retry": true, + "expired_key_can_be_reused": true, + "failovers": [ + { + "confirmed_writes": 217, + "successful_reads": 209, + "errors": 8, + "successful_rps": 28.29, + "first_success_after_action_seconds": 3.881, + "action": { + "finished": 3.821 + }, + "observations": [ + { + "at": 0.007, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.062, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.129, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.183, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.271, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.333, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.392, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.454, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.52, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.586, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.649, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.713, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.766, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.829, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.888, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.959, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.047, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.116, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.181, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.246, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.312, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.376, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.431, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.487, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.545, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.604, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.66, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.721, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.793, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.897, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.996, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.098, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.198, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.291, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.35, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.408, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.492, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.605, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.694, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.796, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.902, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.963, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.034, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.105, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.163, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.223, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.287, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.348, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.406, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.464, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.517, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.577, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.65, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.726, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.799, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.881, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.95, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.017, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.089, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.153, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.221, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.283, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.351, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.416, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.488, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.558, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.633, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.718, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.784, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.863, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.93, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.987, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.083, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.179, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.247, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.309, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.374, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.45, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.526, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.592, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.655, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.713, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.78, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.853, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.94, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.013, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.091, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.196, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.296, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.397, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.502, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.599, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.701, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.763, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.82, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.884, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.951, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.025, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.092, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.164, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.237, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.296, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.357, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.424, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.488, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.554, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.615, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.681, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.746, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.813, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.879, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.947, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.013, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.08, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.142, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.228, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.294, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.355, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.42, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.487, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.549, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.614, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.69, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.765, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.842, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.912, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.986, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.05, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.115, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.195, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.271, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.331, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.397, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.463, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.533, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.596, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.656, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.718, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.78, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.846, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.913, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.974, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.046, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.104, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.164, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.229, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.296, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.357, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.416, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.477, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.543, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.617, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.691, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.763, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.833, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.903, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.969, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.044, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.12, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.194, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.258, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.328, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.386, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.441, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.508, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.572, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.637, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.696, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.761, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.826, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.892, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.961, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.027, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.088, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.168, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.231, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.298, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.361, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.42, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.483, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.553, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.613, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.672, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.735, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.804, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.872, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.941, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.016, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.083, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.149, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.222, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.293, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.365, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.429, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.488, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.548, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.612, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.679, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.746, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.81, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.867, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.93, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.998, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.064, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.151, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.216, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.292, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.356, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.425, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.49, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.567, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.635, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.701, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.768, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.836, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.902, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.974, + "instance": "6e400225cb214074bf1ffd479b9a828e" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "8bbd861211fd", + "pid": 59296, + "started_at": "2026-11-15T20:21:21.104054635Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "079d9eff1d0c4c8cbc0fc0d4b89cb455" + } + }, + { + "confirmed_writes": 214, + "successful_reads": 208, + "errors": 7, + "successful_rps": 28.03, + "first_success_after_action_seconds": 3.725, + "action": { + "finished": 3.682 + }, + "observations": [ + { + "at": 0.017, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.088, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.155, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.215, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.276, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.337, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.406, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.473, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.529, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.589, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.642, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.7, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.754, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.812, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.865, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.939, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.046, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.133, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.239, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.35, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.454, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.54, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.659, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.775, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.853, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.954, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.053, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.151, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.242, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.347, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.444, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.547, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.645, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.702, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.771, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.833, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.896, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.969, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.035, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.097, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.161, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.237, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.294, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.355, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.412, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.473, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.543, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.606, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.725, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.814, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.902, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.977, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.038, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.098, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.163, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.221, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.279, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.346, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.407, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.476, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.546, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.61, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.675, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.739, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.804, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.861, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.919, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.979, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.048, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.115, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.181, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.249, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.324, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.393, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.465, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.531, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.598, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.665, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.732, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.814, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.884, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.947, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.011, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.073, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.136, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.204, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.273, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.34, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.399, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.458, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.518, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.581, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.645, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.711, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.77, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.832, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.898, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.972, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.039, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.113, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.18, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.25, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.316, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.383, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.45, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.522, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.588, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.662, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.732, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.81, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.878, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.951, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.019, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.079, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.145, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.212, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.281, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.347, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.415, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.481, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.55, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.615, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.679, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.744, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.81, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.876, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.94, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.002, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.066, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.182, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.288, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.379, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.49, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.581, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.68, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.746, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.849, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.92, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.979, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.043, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.101, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.16, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.223, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.287, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.35, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.416, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.484, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.556, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.626, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.697, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.762, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.828, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.885, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.956, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.02, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.086, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.148, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.212, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.278, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.337, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.403, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.461, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.525, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.598, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.672, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.742, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.817, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.889, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.978, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.046, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.112, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.177, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.244, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.313, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.38, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.45, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.524, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.594, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.666, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.74, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.813, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.874, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.934, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.995, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.053, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.12, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.188, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.255, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.319, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.385, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.445, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.511, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.569, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.63, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.687, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.752, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.822, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.89, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.971, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.038, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.099, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.164, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.228, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.294, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.364, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.43, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.498, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.564, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.631, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.694, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.759, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.829, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.897, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.967, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "ebc7e4c3838d", + "pid": 59353, + "started_at": "2026-11-15T20:21:21.27865926Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "6e400225cb214074bf1ffd479b9a828e" + } + } + ], + "rolling": { + "confirmed_writes": 618, + "successful_reads": 617, + "errors": 35, + "successful_rps": 27.33, + "first_success_after_action_seconds": 40.902, + "action": { + "finished": 40.842 + }, + "observations": [ + { + "at": 0.022, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.102, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.159, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.225, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.291, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.363, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.456, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.989, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.098, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.157, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.224, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.286, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.349, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.407, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.463, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.528, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.588, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.65, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.719, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.786, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.843, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.898, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.955, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.009, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.074, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.131, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.198, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.259, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.333, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.432, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.503, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.578, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.641, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.73, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.794, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.858, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.916, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.976, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.106, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.175, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.24, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.303, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.368, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.434, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.504, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.578, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.645, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.705, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.769, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.835, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.902, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.969, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.034, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.096, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.168, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.236, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.3, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.366, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.426, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.482, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.547, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.619, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.686, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.752, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.814, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.873, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.928, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.984, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.049, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.113, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.176, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.238, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.297, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.357, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.424, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.483, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.543, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.612, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.673, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.745, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.814, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.879, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.946, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.014, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.076, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.149, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.21, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.273, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.33, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.395, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.453, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.521, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.582, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.654, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.724, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.79, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.853, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.925, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.997, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.163, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.262, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.361, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.463, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.573, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.689, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.746, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.808, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.868, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.934, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.001, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.074, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.143, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.208, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.272, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.352, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.414, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.475, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.531, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.598, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.687, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.772, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.84, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.897, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.959, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.022, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.083, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.149, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.21, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.28, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.347, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.405, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.462, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.525, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.587, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.653, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.718, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.784, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.851, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.913, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.982, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.049, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.107, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.173, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.234, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.296, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.358, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.421, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.49, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.558, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.624, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.684, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.754, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.824, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.884, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.943, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.007, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.076, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.139, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.198, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.276, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.349, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.412, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.475, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.541, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.605, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.665, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.736, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.794, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.851, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.915, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.984, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.046, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.116, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.174, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.24, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.311, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.382, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.453, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.511, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.568, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.628, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.695, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.801, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.863, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.932, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.997, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.06, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.123, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.187, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.251, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.315, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.387, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.45, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.52, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.585, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.648, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.734, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 13.814, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 13.874, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.941, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 14.01, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 14.075, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.139, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 14.2, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 14.265, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.328, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 14.394, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 14.465, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.528, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 14.609, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 14.689, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.757, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 14.842, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 14.928, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.995, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 15.059, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 15.128, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 15.19, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 15.251, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 15.321, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 15.39, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 15.458, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 15.528, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 15.596, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 15.664, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 15.725, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 15.79, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 15.908, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 15.99, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 16.051, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 16.122, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 16.185, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 16.253, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 16.317, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 16.382, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 16.45, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 16.51, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 16.579, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 16.65, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 16.711, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 16.789, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 16.862, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 16.932, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 16.995, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 17.07, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 17.138, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 17.207, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 17.278, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 17.343, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 17.432, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 18.585, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 18.663, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 18.737, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 18.81, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 18.879, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 18.956, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.028, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.092, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.163, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.229, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.303, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.365, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.433, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.493, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.553, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.611, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.679, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.744, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.815, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.882, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.945, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.012, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.075, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.134, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.201, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.267, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.337, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.399, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.46, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.523, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.59, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.661, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.731, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.795, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.863, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.924, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.996, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.061, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.125, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.193, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.263, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.332, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.401, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.475, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.548, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.605, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.663, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.729, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.79, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.854, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.921, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.981, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 22.04, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 22.102, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 22.159, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.232, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.305, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.371, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.444, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.514, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.573, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.639, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.71, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.779, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.844, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.91, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.968, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.031, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.096, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.158, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.224, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.281, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.351, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.446, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.547, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.645, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.747, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.845, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.947, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.01, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.096, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.182, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.25, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.309, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.48, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.548, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.608, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.683, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.75, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.818, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.881, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.952, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.019, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.083, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.146, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.21, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.274, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.342, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.412, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.481, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.55, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.619, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.684, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.752, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.816, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.876, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.943, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.009, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.075, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.136, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.206, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.269, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.339, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.401, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.459, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.518, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.589, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.659, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.718, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.776, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.844, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.91, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.994, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.054, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.118, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.182, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.249, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.308, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.374, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.432, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.492, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.551, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.618, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.685, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.755, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.817, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.885, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.964, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.045, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.116, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.196, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.266, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.351, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.427, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.504, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.625, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.714, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.816, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.909, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 29.004, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 29.171, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 29.406, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 29.522, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 29.705, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 29.909, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.121, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.213, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.31, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.383, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.444, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.51, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.578, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.639, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.698, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.775, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.84, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.907, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.968, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.031, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.094, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.156, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.238, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.309, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.384, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.464, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.538, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.612, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.693, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.763, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.836, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.911, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.981, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.05, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.107, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.182, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.253, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.313, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.385, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.443, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.512, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.578, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.648, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.71, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.773, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.845, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.929, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.993, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 33.179, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 33.277, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 33.36, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 33.433, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 33.496, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 33.578, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 33.643, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 33.706, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 33.769, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 33.837, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 33.9, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 33.968, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 34.041, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 34.117, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 34.245, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 34.31, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 34.402, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 34.47, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 34.533, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 34.6, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 34.671, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 34.737, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 34.803, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 34.871, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 34.943, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 35.008, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 35.076, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 35.142, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 35.209, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 35.267, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 35.333, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 35.401, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 35.46, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 35.532, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 35.599, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 35.66, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 35.721, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 35.788, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 35.854, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 35.916, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 35.984, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 36.045, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 36.105, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.175, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 36.237, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 36.301, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.359, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 36.424, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 36.492, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.56, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 36.628, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 36.691, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.753, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.813, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.892, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.963, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.031, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.1, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.171, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.241, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.311, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.383, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.45, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.529, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.609, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.689, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.77, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.843, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.911, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.977, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.043, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.105, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.168, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.231, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.3, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.368, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.433, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.494, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.568, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.628, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.694, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.763, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.829, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.895, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.961, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.03, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.096, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.175, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.241, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.301, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.366, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.434, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.493, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.56, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.627, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.692, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.758, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.823, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.893, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.96, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.024, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.093, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.159, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.218, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.284, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.35, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.434, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.497, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.564, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.627, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.69, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.758, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.833, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.902, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.978, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.048, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.11, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.178, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.248, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.311, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.372, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.438, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.511, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.58, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.642, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.712, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.782, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.848, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.913, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.977, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.05, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.108, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.168, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.226, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.284, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.355, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.419, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.484, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.549, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.609, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.677, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.743, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.807, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.878, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.952, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.029, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.095, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.159, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.227, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.297, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.377, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.486, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.584, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.679, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.779, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.876, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.943, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.009, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.077, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.144, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.213, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.277, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.343, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.405, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.476, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.546, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.61, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.687, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.754, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.812, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.904, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "before": [ + { + "role": "api", + "container": "8bbd861211fd", + "pid": 99583, + "started_at": "2026-11-15T20:28:29.115127097Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "role": "api", + "container": "ebc7e4c3838d", + "pid": 1593, + "started_at": "2026-11-15T20:29:01.248059875Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "role": "broker", + "container": "55ab9f986bae", + "pid": 58562, + "started_at": "2026-11-15T20:21:12.353588672Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "7014391ecf89", + "pid": 58493, + "started_at": "2026-11-15T20:21:12.04480938Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "fa3c50f83a1f", + "pid": 58431, + "started_at": "2026-11-15T20:21:11.739337255Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "242a49094f0d", + "pid": 60206, + "started_at": "2026-11-15T20:21:33.497731001Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "61281" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "932bb6635829", + "pid": 58277, + "started_at": "2026-11-15T20:21:11.12347638Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "6f6af6bdcd00", + "pid": 59239, + "started_at": "2026-11-15T20:21:20.927591635Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "a4dba713c111", + "pid": 58990, + "started_at": "2026-11-15T20:21:17.651360383Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "ca00d9061bb8", + "pid": 58361, + "started_at": "2026-11-15T20:21:11.435409755Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "1963232f6596", + "pid": 59409, + "started_at": "2026-11-15T20:21:21.488607385Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ], + "after": [ + { + "role": "api", + "container": "b8473993e183", + "pid": 2444, + "started_at": "2026-11-15T20:29:17.8060713Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "role": "api", + "container": "5afc95b85319", + "pid": 3409, + "started_at": "2026-11-15T20:29:36.574748503Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "role": "broker", + "container": "55ab9f986bae", + "pid": 58562, + "started_at": "2026-11-15T20:21:12.353588672Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "7014391ecf89", + "pid": 58493, + "started_at": "2026-11-15T20:21:12.04480938Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "fa3c50f83a1f", + "pid": 58431, + "started_at": "2026-11-15T20:21:11.739337255Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "242a49094f0d", + "pid": 60206, + "started_at": "2026-11-15T20:21:33.497731001Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "61281" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "932bb6635829", + "pid": 58277, + "started_at": "2026-11-15T20:21:11.12347638Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "6f6af6bdcd00", + "pid": 59239, + "started_at": "2026-11-15T20:21:20.927591635Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "a4dba713c111", + "pid": 58990, + "started_at": "2026-11-15T20:21:17.651360383Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "ca00d9061bb8", + "pid": 58361, + "started_at": "2026-11-15T20:21:11.435409755Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "1963232f6596", + "pid": 59409, + "started_at": "2026-11-15T20:21:21.488607385Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ] + }, + "replay_after_replacement": true, + "public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 123, \"failed_attempts\": 2, \"first_recovery_seconds\": 2.6}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.", + "after_persistence": [ + { + "role": "api", + "container": "b8473993e183", + "pid": 8683, + "started_at": "2026-11-15T20:30:59.216457638Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "e61d9c18db8d4153912643f82ea3fd9e" + }, + { + "role": "api", + "container": "5afc95b85319", + "pid": 8607, + "started_at": "2026-11-15T20:30:58.937054013Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "e528375076554fa4b5c9293a17332121" + }, + { + "role": "broker", + "container": "55ab9f986bae", + "pid": 8273, + "started_at": "2026-11-15T20:30:57.728498804Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "7014391ecf89", + "pid": 8292, + "started_at": "2026-11-15T20:30:57.796289054Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "fa3c50f83a1f", + "pid": 58431, + "started_at": "2026-11-15T20:21:11.739337255Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "242a49094f0d", + "pid": 8371, + "started_at": "2026-11-15T20:30:57.957485721Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "61281" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "932bb6635829", + "pid": 8411, + "started_at": "2026-11-15T20:30:57.988178888Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "6f6af6bdcd00", + "pid": 59239, + "started_at": "2026-11-15T20:21:20.927591635Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "a4dba713c111", + "pid": 8870, + "started_at": "2026-11-15T20:31:00.647972291Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "ca00d9061bb8", + "pid": 8267, + "started_at": "2026-11-15T20:30:57.780225846Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "1963232f6596", + "pid": 10016, + "started_at": "2026-11-15T20:31:22.563564969Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ] + }, + "api_worker_relay_reconciler_runtime_hashes_and_revision_match": true, + "fsck": "", + "checks_completed": true, + "result": "PASS" +} diff --git a/scripts/verify_pipeline.py b/scripts/verify_pipeline.py index e327f5fc..28e93854 100644 --- a/scripts/verify_pipeline.py +++ b/scripts/verify_pipeline.py @@ -86,14 +86,20 @@ def duplicate(job, count=5): ) -def inspect_effect(job): +def inspect_effect(job, *, pending=False): return inside(f"""with db.connection() as c: j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone() v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall() p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n'] e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall() keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")] -print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e}},default=str))""") +pending_entries=[] +if {pending!r}: + for item in queue.xpending_range(STREAM,GROUP,'-','+',1000): + entry=item['message_id'] + messages=queue.xrange(STREAM,min=entry,max=entry) + if messages and json.loads(messages[0][1].get('job','{{}}')).get('job_id')==str(j['id']): pending_entries.append(entry) +print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e,'pending_stream_entries':pending_entries}},default=str))""") def main(): @@ -366,7 +372,9 @@ def main(): marker = inject(j, phase, "wait") start("worker") wait(lambda marker=marker: hit(marker)) - at_crash = inspect_effect(j) + at_crash = inspect_effect( + j, pending=phase == "after_commit" + ) assert at_crash["job"]["state"] == ( "ready" if phase == "after_commit" @@ -378,6 +386,14 @@ def main(): assert len(at_crash["physical_keys"]) == ( 0 if phase == "before_effect" else 4 ) + if phase == "after_commit": + assert at_crash["pending_stream_entries"], ( + "Result must be committed while notification is still unacknowledged" + ) + assert not any( + e["event"] == "child_finished" + for e in at_crash["events"] + ) compose("kill", "-s", "SIGKILL", "worker") start("worker") effect = confirm(a, j) diff --git a/src/worker.py b/src/worker.py index 274684e3..e9530911 100644 --- a/src/worker.py +++ b/src/worker.py @@ -315,7 +315,8 @@ def consume(slot): context = multiprocessing.get_context("spawn") cursor = "0-0" timeout = integer("JOB_TIMEOUT_SECONDS", 20, 2, 117) - if integer("JOB_LEASE_SECONDS", 23, 3, 120) <= timeout: + lease = integer("JOB_LEASE_SECONDS", 23, 3, 120) + if lease <= timeout: raise ValueError("Lease must exceed child timeout") try: while True: @@ -329,7 +330,8 @@ def consume(slot): STREAM, GROUP, consumer, - min_idle_time=3000, + # Do not steal an entry from a consumer inside its live budget. + min_idle_time=lease * 1000, start_id=cursor, count=1, )