From eb6d229afb1437cd0f8b5e2ee0ca384d584ddd8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 23:18:33 +0300 Subject: [PATCH] test(pipeline): record verified limits and complete cluster recovery evidence --- REPORT.md | 18 + evidence/cluster-first.json | 5162 +++++++++++++++++++++++++++++++++++ scripts/verify_pipeline.py | 19 +- src/worker.py | 15 + 4 files changed, 5199 insertions(+), 15 deletions(-) create mode 100644 evidence/cluster-first.json diff --git a/REPORT.md b/REPORT.md index 8c0318b2..83821142 100644 --- a/REPORT.md +++ b/REPORT.md @@ -286,3 +286,21 @@ private/IAM/otherAPI. Это целевой повтор, а не19cases пол проходит handoff predicate. Fixture metadata failed состарены SQL отдельно, но возраст S3 LastModified не подменялся. Cleanup fixtures синтетические и не обещают восстановление bytes из checksum. + +Для финальной проверки worker перед fault hook сохраняет diagnostic event +worker_resource_limits с фактическим RLIMIT_AS и cgroup memory.max/cpu.max. +Сбор показаний самим child исключает влияние медленного внешнего DockerCLI +на12s test budget. Это диагностическое добавление также проверяется новым +образом из чистого clone; предыдущие HTTP/source evidence не переименованы. + +Первый cluster receipt [cluster-first-failed.json](evidence/cluster-first-failed.json) +сохраняет успешные отказы двухAPI. Rolling занял дольше45s окна, поэтому +генератор завершился сразу с action и не получил post-action observation. +Verifier теперь обязательно наблюдает ещё2s после action, сохраняя все +проверки данных и failover≤10s. Это исправление наблюдения, не ослабление SLA. + +[evidence/cluster-first.json](evidence/cluster-first.json): complete PASS; +API failovers (writes/errors/recovery seconds): 214/6/3.086, 216/8/2.762. +Rolling: 730 confirmed writes, 18 transient errors; все исходныеAPI заменены, данные/replay сохранены. +Public failover/persistence PASS; StartedAt PG/cache/S3/broker/LB изменились, +прежний token и file сохранились. Это не zero-error rolling. diff --git a/evidence/cluster-first.json b/evidence/cluster-first.json new file mode 100644 index 00000000..e642d431 --- /dev/null +++ b/evidence/cluster-first.json @@ -0,0 +1,5162 @@ +{ + "date": "2026-11-15T20:13:14.393195+00:00", + "source_commit": "ef619490f23877adbce37c9012496d01128bae4a", + "complete": true, + "before": [ + { + "role": "api", + "container": "638206e980b8", + "pid": 31709, + "started_at": "2026-11-15T20:09:56.153775471Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "c5f6207268fe429cb5ead0adc052bc01" + }, + { + "role": "api", + "container": "b5d2b8c25029", + "pid": 33126, + "started_at": "2026-11-15T20:10:27.099626888Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "role": "broker", + "container": "5d3ebf6f87ff", + "pid": 77503, + "started_at": "2026-11-15T19:51:03.049505001Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "6ad17613e620", + "pid": 60587, + "started_at": "2026-11-15T19:17:14.204239507Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "c6fb99c3ddbc", + "pid": 60326, + "started_at": "2026-11-15T19:45:43.876610632Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "c60ecc6c94f4", + "pid": 63147, + "started_at": "2026-11-15T19:17:36.952605129Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "76efa23ba942", + "pid": 60650, + "started_at": "2026-11-15T19:17:14.476492674Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "a990e3899324", + "pid": 27505, + "started_at": "2026-11-15T20:08:42.302428089Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "c8273b4552c9", + "pid": 98443, + "started_at": "2026-11-15T19:58:04.386027252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "de0c05e2e665", + "pid": 60843, + "started_at": "2026-11-15T19:17:15.006133424Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "a89e2cf493d9", + "pid": 18369, + "started_at": "2026-11-15T20:05:23.317701469Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ], + "parallel_replay": { + "requests": 30, + "messages": 1, + "instances": [ + "71696073e468446ea13d821b111ad508", + "c5f6207268fe429cb5ead0adc052bc01" + ] + }, + "atomic_rollback_and_5xx_retry": true, + "expired_key_can_be_reused": true, + "failovers": [ + { + "confirmed_writes": 214, + "successful_reads": 208, + "errors": 6, + "successful_rps": 27.94, + "first_success_after_action_seconds": 3.086, + "action": { + "finished": 3.081 + }, + "observations": [ + { + "at": 0.008, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.065, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.184, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.256, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.368, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.469, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.57, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.663, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.723, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.778, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.843, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.907, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.968, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.059, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.162, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.264, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.358, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.469, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.527, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.617, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.707, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.818, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.918, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.014, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.111, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.212, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.283, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.34, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.394, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.448, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.51, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.571, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.63, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.691, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.75, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.808, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.876, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.945, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.007, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.086, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.164, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.232, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.296, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.365, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.447, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.506, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.57, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.634, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.699, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.759, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.814, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.877, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.944, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.008, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.073, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.152, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.232, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.293, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.351, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.413, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.474, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.542, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.608, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.677, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.745, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.808, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.862, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.93, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.988, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.054, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.112, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.174, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.243, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.308, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.379, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.452, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.519, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.595, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.669, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.742, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.807, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.871, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.936, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.999, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.057, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.122, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.18, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.244, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.312, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.375, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.441, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.508, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.574, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.642, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.758, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.828, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.888, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.962, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.023, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.092, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.152, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.213, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.28, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.341, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.409, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.471, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.543, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.609, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.678, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.747, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.809, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.878, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.963, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.047, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.125, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.194, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.258, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.32, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.395, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.458, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.528, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.592, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.662, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.726, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.795, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.859, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.929, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.999, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.072, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.133, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.198, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.256, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.326, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.392, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.457, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.523, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.608, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.711, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.815, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.915, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.016, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.11, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.208, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.319, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.39, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.456, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.569, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.63, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.69, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.757, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.814, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.874, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.935, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.997, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.064, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.131, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.199, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.263, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.339, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.407, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.469, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.536, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.594, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.663, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.733, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.806, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.873, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.945, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.011, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.085, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.154, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.217, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.289, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.355, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.419, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.486, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.547, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.607, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.668, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.736, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.797, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.857, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.922, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.984, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.049, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.126, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.189, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.248, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.307, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.365, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.428, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.496, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.558, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.624, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.696, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.771, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.846, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.918, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.988, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.062, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.142, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.21, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.276, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.342, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.416, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.483, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.552, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.611, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.67, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.734, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.794, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.857, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.925, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.994, + "instance": "71696073e468446ea13d821b111ad508" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "638206e980b8", + "pid": 31709, + "started_at": "2026-11-15T20:09:56.153775471Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "c5f6207268fe429cb5ead0adc052bc01" + } + }, + { + "confirmed_writes": 216, + "successful_reads": 209, + "errors": 8, + "successful_rps": 28.18, + "first_success_after_action_seconds": 2.762, + "action": { + "finished": 2.701 + }, + "observations": [ + { + "at": 0.018, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.084, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.15, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.215, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.28, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.343, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.403, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.459, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.525, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.582, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.636, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.694, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.751, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.817, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.873, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.935, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.005, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.108, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.204, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.316, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.41, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.468, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.574, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.677, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.779, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.87, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.97, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.031, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.1, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.164, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.223, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.278, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.339, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.394, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.447, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.505, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.561, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.619, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.687, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.762, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.832, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.888, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.946, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.009, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.072, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.139, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.209, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.275, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.338, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.431, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.487, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.542, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.608, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.673, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.738, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.796, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.854, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.914, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.973, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.045, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.107, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.169, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.241, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.312, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.38, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.446, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.504, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.563, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.628, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.69, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.759, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.83, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.89, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.953, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.014, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.071, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.139, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.204, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.272, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.343, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.407, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.489, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.556, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.624, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.693, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.767, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.838, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.904, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.96, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.025, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.092, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.157, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.234, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.301, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.367, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.435, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.499, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.566, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.635, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.702, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.765, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.837, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.931, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.0, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.084, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.156, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.22, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.285, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.347, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.41, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.473, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.568, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.634, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.701, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.767, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.842, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.914, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.987, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.054, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.123, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.19, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.261, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.32, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.386, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.446, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.51, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.581, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.648, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.72, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.787, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.856, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.923, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.986, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.047, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.116, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.181, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.248, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.311, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.371, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.459, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.659, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.765, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.864, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.954, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.055, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.164, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.26, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.363, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.425, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.487, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.557, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.625, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.698, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.767, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.833, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.904, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.974, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.033, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.091, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.155, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.225, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.286, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.355, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.426, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.49, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.554, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.613, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.68, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.748, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.815, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.878, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.945, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.009, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.073, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.137, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.198, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.255, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.315, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.382, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.443, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.513, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.577, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.653, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.723, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.795, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.866, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.935, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.01, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.135, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.201, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.269, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.339, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.409, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.475, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.545, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.615, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.681, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.752, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.822, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.885, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.958, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.024, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.101, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.185, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.253, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.314, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.381, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.445, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.513, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.572, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.64, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.704, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.765, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.829, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.891, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.959, + "instance": "5877164629d44631b753484921940317" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "b5d2b8c25029", + "pid": 33126, + "started_at": "2026-11-15T20:10:27.099626888Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "71696073e468446ea13d821b111ad508" + } + } + ], + "rolling": { + "confirmed_writes": 730, + "successful_reads": 728, + "errors": 18, + "successful_rps": 27.16, + "first_success_after_action_seconds": 51.552, + "action": { + "finished": 51.507 + }, + "observations": [ + { + "at": 0.023, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.087, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.156, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.22, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.278, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.343, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.402, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.466, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.53, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.586, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.646, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.702, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.758, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.827, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.909, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.002, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.068, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.152, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.232, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.327, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.422, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.501, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.595, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.684, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.76, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.86, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.952, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.478, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.561, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.671, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.748, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.808, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.88, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.945, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.007, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.067, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.129, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.196, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.259, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.317, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.384, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.443, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.498, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.558, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.644, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.699, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.763, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.819, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.883, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.948, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.013, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.083, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.152, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.222, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.281, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.341, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.4, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.463, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.54, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.626, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.708, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.769, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.835, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.895, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.964, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.022, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.087, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.153, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.22, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.285, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.347, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.412, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.475, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.535, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.604, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.675, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.744, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.808, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.872, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.934, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.992, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.052, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.118, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.175, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.234, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.296, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.361, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.427, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.492, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.548, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.615, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.684, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.749, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.899, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.033, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.16, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.622, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.931, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.048, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.153, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.24, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.347, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.443, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.537, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.638, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.709, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.778, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.846, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.913, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.973, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.035, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.103, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.172, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.235, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.298, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.367, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.433, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.49, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.546, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.613, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.68, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.741, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.81, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.873, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.943, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.005, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.067, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.136, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.202, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.272, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.338, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.404, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.469, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.534, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.602, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.675, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.737, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.802, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.862, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.919, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.984, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.05, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.118, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.188, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.253, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.33, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.408, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.473, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.539, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.601, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.669, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.737, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.816, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.92, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.01, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.088, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.155, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.226, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.349, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.438, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.503, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.568, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.636, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.705, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.778, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.844, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.903, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.964, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.03, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.095, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.167, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.239, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.306, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.376, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.441, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.513, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.585, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.643, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.705, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.774, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.836, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.903, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.974, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.044, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.104, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.175, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.236, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.294, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.351, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.412, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.48, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.547, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.618, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.688, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.758, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.834, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.921, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 14.998, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 15.061, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 15.13, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 15.196, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 15.263, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 15.337, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 15.406, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 15.475, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 15.539, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 15.608, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 15.665, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 15.723, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 15.783, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 15.851, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 15.931, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 16.002, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 16.071, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 16.139, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 16.202, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 16.27, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 16.34, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 16.4, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 16.468, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 16.539, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 16.604, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 16.679, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 16.746, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 16.809, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 16.87, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 16.929, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 16.994, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 17.061, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 17.124, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 17.191, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 17.26, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 17.33, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 17.4, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 17.465, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 17.528, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 17.588, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 17.649, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 17.712, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 17.778, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 17.836, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 17.903, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 17.986, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 18.052, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 18.133, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 18.202, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 18.278, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 18.344, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 18.411, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 18.479, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 18.554, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 18.689, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 18.82, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 18.953, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 19.08, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 19.144, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 19.249, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 19.382, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 19.446, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 19.518, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 19.593, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 19.665, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 19.732, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 19.793, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 19.862, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 19.929, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 20.006, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 20.088, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 20.158, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 20.24, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 20.308, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 20.369, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 20.428, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 20.492, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 20.552, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 20.613, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 20.678, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 20.74, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 20.805, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 20.872, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 20.937, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 20.997, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 21.054, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 21.112, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 21.177, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 21.236, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 21.304, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 21.365, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 21.428, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 21.485, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 21.548, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 21.607, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 21.666, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 21.737, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 21.804, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 21.862, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 21.92, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 21.987, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 22.055, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 22.139, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 22.211, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 22.276, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 22.334, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 22.399, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 22.456, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 22.517, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 22.584, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 22.653, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 22.723, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 22.796, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 22.87, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 22.936, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 22.998, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 23.065, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 23.131, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 23.196, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 23.264, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 23.33, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 23.388, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 23.448, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 23.514, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 23.587, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 23.713, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 23.776, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 23.88, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 23.942, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 24.01, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 24.068, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 24.135, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 24.203, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 24.274, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 24.343, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 24.412, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 24.469, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 24.533, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 24.6, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 24.656, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 24.714, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 24.78, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 24.844, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 24.911, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 24.976, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 25.045, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 25.113, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 25.188, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 25.264, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 25.34, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 25.424, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 25.489, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 25.548, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 25.617, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 25.687, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 25.753, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 25.819, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 25.885, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 25.953, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 26.02, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 26.083, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 26.156, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 26.23, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 26.302, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 26.369, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 26.435, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 26.5, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 26.564, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 26.624, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 26.69, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 26.756, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 26.818, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 26.884, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 26.947, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 27.012, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.073, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 27.136, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 27.197, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.257, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 27.322, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 27.395, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.476, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 27.556, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 27.62, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.695, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.851, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.926, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.998, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.062, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.14, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.248, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.314, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.399, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.476, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.557, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.637, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.717, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.791, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.865, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.937, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.004, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.062, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.131, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.195, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.266, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.331, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.404, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.484, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.581, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.685, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.788, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.895, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.99, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.089, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.155, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.219, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.301, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.369, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.437, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.501, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.564, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.631, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.698, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.767, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.838, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.918, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.99, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.068, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.143, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.221, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.285, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.352, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.418, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.478, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.54, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.604, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.676, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.742, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.808, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.873, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.935, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.009, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.073, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.147, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.226, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.322, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.428, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.511, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.595, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.686, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.768, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.839, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.897, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.96, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.022, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.092, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.152, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.255, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.324, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.392, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.467, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.532, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.615, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.693, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.781, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.856, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.922, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.984, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.053, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.174, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.276, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.383, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.48, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.577, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.672, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.767, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.864, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.98, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.047, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.118, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.201, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.271, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.336, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.417, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.482, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.548, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.618, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.678, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.74, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.811, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.896, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.982, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.051, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.12, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.189, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.265, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.325, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.392, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.489, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.551, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.623, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.69, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.754, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.821, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.888, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.966, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.023, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.09, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.153, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.218, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.276, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.337, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.397, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.466, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.534, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.602, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.666, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.724, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.785, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.843, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.906, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.974, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.039, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.11, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.184, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.25, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.317, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.383, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.443, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.512, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.597, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.664, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.741, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.809, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.879, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.946, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.015, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.081, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.137, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.195, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.26, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.318, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.378, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.439, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.503, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.571, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.643, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.718, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.777, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.836, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.896, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.96, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.025, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.082, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.144, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.284, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.383, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.489, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.585, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.689, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.78, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.84, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.917, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.991, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.062, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.124, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.182, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.25, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.32, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.387, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.453, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.519, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.582, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.643, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.738, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.801, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.862, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.929, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.994, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.066, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.125, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.189, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.252, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.316, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.382, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.445, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.514, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.585, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.663, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.723, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.787, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.854, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.919, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.981, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.052, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.126, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.186, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.251, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.315, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.38, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.452, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.514, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.579, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.646, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.705, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.764, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.831, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.892, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.958, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.016, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.076, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.149, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.215, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.283, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.342, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.43, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 44.492, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 44.56, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.62, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 44.69, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 44.753, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.824, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 44.893, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 44.959, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 45.028, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 45.121, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 45.181, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 45.272, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 45.331, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 45.394, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 45.476, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 45.547, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 45.613, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 45.698, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 45.768, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 45.835, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 45.902, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 45.96, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 46.029, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 46.09, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 46.149, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 46.208, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 46.274, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 46.341, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 46.404, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 46.467, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 46.534, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 46.595, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 46.657, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 46.723, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 46.786, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 46.847, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 46.915, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 46.977, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 47.043, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 47.104, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.161, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 47.228, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 47.294, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.356, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 47.429, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 47.497, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.564, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 47.635, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 47.691, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.772, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.843, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.915, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.982, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 48.044, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 48.107, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.229, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.301, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.371, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.43, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.487, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.557, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.626, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.687, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.755, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.828, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.892, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.965, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.036, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.097, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.164, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.227, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.297, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.371, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.44, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.512, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.576, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.635, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.7, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.773, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.84, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.904, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.97, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.037, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.095, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.153, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.219, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.281, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.35, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.416, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.483, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.552, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.625, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.692, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.761, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.834, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.896, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.961, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.024, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.089, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.154, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.421, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.514, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.582, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.643, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.704, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.772, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.846, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.905, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.971, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.04, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.104, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.17, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.237, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.304, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.37, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.438, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.507, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "before": [ + { + "role": "api", + "container": "638206e980b8", + "pid": 41688, + "started_at": "2026-11-15T20:13:39.959219713Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "5877164629d44631b753484921940317" + }, + { + "role": "api", + "container": "b5d2b8c25029", + "pid": 43471, + "started_at": "2026-11-15T20:14:12.673332131Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "role": "broker", + "container": "5d3ebf6f87ff", + "pid": 77503, + "started_at": "2026-11-15T19:51:03.049505001Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "6ad17613e620", + "pid": 60587, + "started_at": "2026-11-15T19:17:14.204239507Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "c6fb99c3ddbc", + "pid": 60326, + "started_at": "2026-11-15T19:45:43.876610632Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "c60ecc6c94f4", + "pid": 63147, + "started_at": "2026-11-15T19:17:36.952605129Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "76efa23ba942", + "pid": 60650, + "started_at": "2026-11-15T19:17:14.476492674Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "a990e3899324", + "pid": 27505, + "started_at": "2026-11-15T20:08:42.302428089Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "c8273b4552c9", + "pid": 98443, + "started_at": "2026-11-15T19:58:04.386027252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "de0c05e2e665", + "pid": 60843, + "started_at": "2026-11-15T19:17:15.006133424Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "a89e2cf493d9", + "pid": 18369, + "started_at": "2026-11-15T20:05:23.317701469Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ], + "after": [ + { + "role": "api", + "container": "6ca6b7ae13ea", + "pid": 44302, + "started_at": "2026-11-15T20:14:28.278164513Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "role": "api", + "container": "a6c104cad48f", + "pid": 45744, + "started_at": "2026-11-15T20:14:59.195657555Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "role": "broker", + "container": "5d3ebf6f87ff", + "pid": 77503, + "started_at": "2026-11-15T19:51:03.049505001Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "6ad17613e620", + "pid": 60587, + "started_at": "2026-11-15T19:17:14.204239507Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "c6fb99c3ddbc", + "pid": 60326, + "started_at": "2026-11-15T19:45:43.876610632Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "c60ecc6c94f4", + "pid": 63147, + "started_at": "2026-11-15T19:17:36.952605129Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "76efa23ba942", + "pid": 60650, + "started_at": "2026-11-15T19:17:14.476492674Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "a990e3899324", + "pid": 27505, + "started_at": "2026-11-15T20:08:42.302428089Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "c8273b4552c9", + "pid": 98443, + "started_at": "2026-11-15T19:58:04.386027252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "de0c05e2e665", + "pid": 60843, + "started_at": "2026-11-15T19:17:15.006133424Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "a89e2cf493d9", + "pid": 18369, + "started_at": "2026-11-15T20:05:23.317701469Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ] + }, + "replay_after_replacement": true, + "public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 112, \"failed_attempts\": 4, \"first_recovery_seconds\": 3.812}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.", + "after_persistence": [ + { + "role": "api", + "container": "6ca6b7ae13ea", + "pid": 51295, + "started_at": "2026-11-15T20:16:29.70121475Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "f9d06f9380aa472987e637ad3b18840e" + }, + { + "role": "api", + "container": "a6c104cad48f", + "pid": 51341, + "started_at": "2026-11-15T20:16:29.821670166Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "8c9b45de14704aa59c25b46076ac749a" + }, + { + "role": "broker", + "container": "5d3ebf6f87ff", + "pid": 50906, + "started_at": "2026-11-15T20:16:28.119800305Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "6ad17613e620", + "pid": 51018, + "started_at": "2026-11-15T20:16:28.220991221Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "c6fb99c3ddbc", + "pid": 60326, + "started_at": "2026-11-15T19:45:43.876610632Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "c60ecc6c94f4", + "pid": 51053, + "started_at": "2026-11-15T20:16:28.347761055Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "76efa23ba942", + "pid": 51061, + "started_at": "2026-11-15T20:16:28.35964443Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "a990e3899324", + "pid": 27505, + "started_at": "2026-11-15T20:08:42.302428089Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "c8273b4552c9", + "pid": 51509, + "started_at": "2026-11-15T20:16:30.976766958Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "de0c05e2e665", + "pid": 50945, + "started_at": "2026-11-15T20:16:28.170668388Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "a89e2cf493d9", + "pid": 52672, + "started_at": "2026-11-15T20:16:52.907797677Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ] +} diff --git a/scripts/verify_pipeline.py b/scripts/verify_pipeline.py index c92da95d..e327f5fc 100644 --- a/scripts/verify_pipeline.py +++ b/scripts/verify_pipeline.py @@ -413,22 +413,11 @@ def main(): marker = inject(j, "before_effect", "hang") start("worker") wait(lambda: hit(marker)) - cid = compose("ps", "-q", "worker").split()[0] - code = """import json,pathlib,re - limits=[] - for p in pathlib.Path('/proc').glob('[0-9]*/limits'): - try: - line=next(x for x in p.read_text().splitlines() if x.startswith('Max address space')) - if re.search(r'805306368\\s+805306368',line): limits.append({'pid':p.parent.name,'limit':line}) - except (FileNotFoundError,PermissionError,StopIteration,ProcessLookupError): pass - root=pathlib.Path('/sys/fs/cgroup') - print(json.dumps({'bounded_children':limits,'cgroup':{n:(root/n).read_text().strip() for n in ['memory.max','memory.current','memory.peak','cpu.max','pids.current']}})) - """ - limits = json.loads( - command(["docker", "exec", cid, "python", "-c", code]) + limits = inside( + f"with db.connection() as c: r=c.execute(\"SELECT detail FROM pipeline_events WHERE job_id=%s AND event='worker_resource_limits' ORDER BY id DESC LIMIT 1\",({j!r},)).fetchone()\nprint(json.dumps(r['detail']))" ) - assert 1 <= len(limits["bounded_children"]) <= 2 - assert limits["cgroup"]["memory.max"] == "1610612736" + assert limits["address_space_bytes"] == [805306368, 805306368] + assert limits["memory.max"] == "1610612736" effect = confirm(a, j) assert any( e["event"] == "retry" diff --git a/src/worker.py b/src/worker.py index 70fc9fc2..274684e3 100644 --- a/src/worker.py +++ b/src/worker.py @@ -7,6 +7,7 @@ import multiprocessing import resource import threading import time +from pathlib import Path from uuid import uuid4 import redis @@ -122,6 +123,20 @@ def process(job_id, fence, sender): if not row: sender.send({"result": "stale"}) return + # Record limits before fault hooks: a slow external inspector must not + # consume the child timeout while attempting to observe the limit. + cgroup = Path("/sys/fs/cgroup") + limits = { + name: (cgroup / name).read_text().strip() + if (cgroup / name).exists() + else "unavailable" + for name in ["memory.max", "cpu.max"] + } + limits["address_space_bytes"] = list( + resource.getrlimit(resource.RLIMIT_AS) + ) + with db.connection() as conn: + event(conn, job_id, "worker_resource_limits", limits) fault(db, job_id, "before_effect") upstream = store.get(key=row["source_key"]) try: