Files
mtusi2026lab5/evidence/clean-pipeline.json
T

1308 lines
36 KiB
JSON

{
"anonymous_s3_status": 403,
"denied_iam_actions": [
"api:put_object",
"api:delete_object",
"worker:delete_object",
"reconciler:put_object"
],
"cross_replica": {
"uploader": "0a9437289ec643a2bfc1fe79c1409f2b",
"stopped": {
"id": "1cf780371b6e3d5a10b50b91dd1dadddb1371f7f1d2c94bbc0437adc45ae1f4d",
"started_at": "2026-10-17T17:54:22.316986762Z",
"pid": 0,
"running": false,
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {}
},
"reader": "52ae54c070824ef39ea11614a3a3684a"
},
"track": 2,
"delivery": "at-least-once",
"effect_key": [
"attachment_id",
"pipeline_version",
"variant"
],
"worker_concurrency": 2,
"prefetch_per_slot": 1,
"job_timeout_seconds": 5,
"lease_seconds": 8,
"max_attempts": 5,
"scenarios": [
{
"scenario": "commit_publish_gap_api_restart_broker_sigkill",
"attachment_id": "bffcca56-87f6-4e5f-9f0b-b246e3216f13",
"job_id": "07853ae0-ebf0-4f59-a835-c6c47bbec488",
"elapsed_seconds": 10.150205792000634,
"final_state": "ready",
"attempts": 1,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:54:21.522737+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259667046-0"
},
"created_at": "2026-10-17 17:54:27.056252+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "00c89838-3f0d-429b-9641-ffd458180ba1",
"attempt": 1
},
"created_at": "2026-10-17 17:54:31.193598+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:54:31.240139+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:54:31.241485+00:00"
}
],
"api_before": [
{
"id": "50f1efb1b58d309cf2f9084d2af4270dbdf26f11b211cee05b02b3d555e1c250",
"started_at": "2026-10-17T17:53:40.323395673Z",
"pid": 82578,
"running": true,
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "2df8aa848dbe43f7b28d10fb379202f8"
},
{
"id": "1cf780371b6e3d5a10b50b91dd1dadddb1371f7f1d2c94bbc0437adc45ae1f4d",
"started_at": "2026-10-17T17:53:40.39698984Z",
"pid": 82634,
"running": true,
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "54220b0dbefb48bfbc470fbf31a6bd96"
}
],
"api_after": [
{
"id": "50f1efb1b58d309cf2f9084d2af4270dbdf26f11b211cee05b02b3d555e1c250",
"started_at": "2026-10-17T17:54:22.360219637Z",
"pid": 86502,
"running": true,
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "52ae54c070824ef39ea11614a3a3684a"
},
{
"id": "1cf780371b6e3d5a10b50b91dd1dadddb1371f7f1d2c94bbc0437adc45ae1f4d",
"started_at": "2026-10-17T17:54:22.316986762Z",
"pid": 86447,
"running": true,
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "0a9437289ec643a2bfc1fe79c1409f2b"
}
],
"broker_before": {
"id": "95c5c9f8fb2d909cada1fa7e3b45f81524d42b83b72b7dba1029b8bf705f3b25",
"started_at": "2026-10-17T17:53:35.222258712Z",
"pid": 81855,
"running": true,
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
"memory": 402653184,
"nano_cpus": 500000000,
"published_ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56385"
}
]
}
},
"broker_after": {
"id": "95c5c9f8fb2d909cada1fa7e3b45f81524d42b83b72b7dba1029b8bf705f3b25",
"started_at": "2026-10-17T17:54:27.551506417Z",
"pid": 87340,
"running": true,
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
"memory": 402653184,
"nano_cpus": 500000000,
"published_ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56385"
}
]
}
},
"broker_entry_ids": [
"1792259667046-0"
],
"message_id": "d121d1ff-ac42-48c1-ba36-6ef3fa377015"
},
{
"scenario": "publish_before_outbox_commit_duplicate",
"attachment_id": "ccf6515f-d80d-40db-be0e-33cc21684fe9",
"job_id": "47da08ce-cbc3-4ef0-abc6-f3275a7f3ef2",
"elapsed_seconds": 14.278877541999464,
"final_state": "ready",
"attempts": 1,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:54:32.071860+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 1,
"phase": "after_publish",
"action": "crash"
},
"created_at": "2026-10-17 17:54:33.771814+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259674388-0"
},
"created_at": "2026-10-17 17:54:34.396026+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "2b004cdb-3083-4ff8-94c9-7374a104896c",
"attempt": 1
},
"created_at": "2026-10-17 17:54:38.179597+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:54:38.233346+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:54:38.234856+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:54:46.175653+00:00"
}
]
},
{
"scenario": "worker_crash_before_effect",
"attachment_id": "36ce0968-4044-4da3-b581-93f4843bc8d6",
"job_id": "3f333283-15ed-44a9-bd1a-2be5ae212c5e",
"elapsed_seconds": 13.600872499999241,
"final_state": "ready",
"attempts": 2,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:54:46.619877+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259686809-0"
},
"created_at": "2026-10-17 17:54:46.813863+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "795b98c1-968d-4c19-ac55-7e1e15e20fe6",
"attempt": 1
},
"created_at": "2026-10-17 17:54:50.189662+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "before_effect",
"action": "crash"
},
"created_at": "2026-10-17 17:54:50.191055+00:00"
},
{
"event": "lease_expired",
"details": {},
"created_at": "2026-10-17 17:54:58.255525+00:00"
},
{
"event": "queued",
"details": {
"reason": "worker_timeout",
"attempt": 1,
"delay_seconds": 1.1620223936252645
},
"created_at": "2026-10-17 17:54:58.256553+00:00"
},
{
"event": "published",
"details": {
"attempt": 2,
"stream_id": "1792259699473-0"
},
"created_at": "2026-10-17 17:54:59.478214+00:00"
},
{
"event": "claimed",
"details": {
"pid": 18,
"fence": "8e2780d0-cf82-4fdc-be8c-9b553794686f",
"attempt": 2
},
"created_at": "2026-10-17 17:55:00.113366+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:55:00.161563+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:55:00.163145+00:00"
}
]
},
{
"scenario": "worker_crash_after_effect",
"attachment_id": "0b374955-a72f-49fa-ba6c-873174a2a445",
"job_id": "4feb58df-db81-47f0-b0e7-0fd99256c8b8",
"elapsed_seconds": 13.586232916999506,
"final_state": "ready",
"attempts": 2,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:55:00.717466+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259700904-0"
},
"created_at": "2026-10-17 17:55:00.905775+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "8a11cf54-ccea-4268-8ee4-00ca0bce64c8",
"attempt": 1
},
"created_at": "2026-10-17 17:55:04.321489+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "after_effect",
"action": "crash"
},
"created_at": "2026-10-17 17:55:04.350788+00:00"
},
{
"event": "lease_expired",
"details": {},
"created_at": "2026-10-17 17:55:12.366261+00:00"
},
{
"event": "queued",
"details": {
"reason": "worker_timeout",
"attempt": 1,
"delay_seconds": 0.8627423719909547
},
"created_at": "2026-10-17 17:55:12.366899+00:00"
},
{
"event": "published",
"details": {
"attempt": 2,
"stream_id": "1792259713389-0"
},
"created_at": "2026-10-17 17:55:13.395682+00:00"
},
{
"event": "claimed",
"details": {
"pid": 23,
"fence": "8db5568d-9e34-4efc-a3b3-7138c347b386",
"attempt": 2
},
"created_at": "2026-10-17 17:55:14.015951+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:55:14.064324+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:55:14.065699+00:00"
}
]
},
{
"scenario": "worker_crash_after_commit",
"attachment_id": "5ab078f1-6c88-4804-b2eb-020ad667ed44",
"job_id": "898833da-cd51-4943-8663-4580448c1984",
"elapsed_seconds": 11.788440333000835,
"final_state": "ready",
"attempts": 1,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:55:14.613047+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259714618-0"
},
"created_at": "2026-10-17 17:55:14.620698+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "3ad27bd3-8c04-477e-be69-8319cf9e703d",
"attempt": 1
},
"created_at": "2026-10-17 17:55:18.200056+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:55:18.246743+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:55:18.248223+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "after_commit",
"action": "crash"
},
"created_at": "2026-10-17 17:55:18.248930+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:55:26.249978+00:00"
}
]
},
{
"scenario": "transient_2",
"attachment_id": "e4302e45-e345-46ac-aab4-ac4f6292aaf2",
"job_id": "e1671b9f-9177-4ba2-9461-a1bef05942d6",
"elapsed_seconds": 7.436319458000071,
"final_state": "ready",
"attempts": 3,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:55:26.814495+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259726857-0"
},
"created_at": "2026-10-17 17:55:26.859168+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "1d7dadee-bc81-46f8-ada0-53600673b332",
"attempt": 1
},
"created_at": "2026-10-17 17:55:30.364647+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:55:30.366081+00:00"
},
{
"event": "queued",
"details": {
"reason": "storage_unavailable",
"attempt": 1,
"delay_seconds": 0.41945831889983076
},
"created_at": "2026-10-17 17:55:30.367279+00:00"
},
{
"event": "published",
"details": {
"attempt": 2,
"stream_id": "1792259730919-0"
},
"created_at": "2026-10-17 17:55:30.921761+00:00"
},
{
"event": "claimed",
"details": {
"pid": 13,
"fence": "53f36327-c72f-4d27-ad8c-db27d561ba3e",
"attempt": 2
},
"created_at": "2026-10-17 17:55:31.556720+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 13,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:55:31.557980+00:00"
},
{
"event": "queued",
"details": {
"reason": "storage_unavailable",
"attempt": 2,
"delay_seconds": 0.988456979754063
},
"created_at": "2026-10-17 17:55:31.559262+00:00"
},
{
"event": "published",
"details": {
"attempt": 3,
"stream_id": "1792259732564-0"
},
"created_at": "2026-10-17 17:55:32.570956+00:00"
},
{
"event": "claimed",
"details": {
"pid": 18,
"fence": "af9be6c6-dd49-4d38-8ae9-e145c1d389e7",
"attempt": 3
},
"created_at": "2026-10-17 17:55:33.193577+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:55:33.245406+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:55:33.246917+00:00"
}
],
"terminal_before_replay": null,
"backoff_delays": [
0.41945831889983076,
0.988456979754063
]
},
{
"scenario": "hang_1",
"attachment_id": "ac601ccb-0c28-4bdc-b3ce-b8b4ad08d8a7",
"job_id": "cdb07fbb-9ff9-45e5-b46f-39ce1159927a",
"elapsed_seconds": 13.753177582999342,
"final_state": "ready",
"attempts": 2,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:55:34.590420+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259734606-0"
},
"created_at": "2026-10-17 17:55:34.608576+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "42ca70c4-57e4-4fbe-a86c-8ccd83c21749",
"attempt": 1
},
"created_at": "2026-10-17 17:55:38.224796+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "before_effect",
"action": "hang"
},
"created_at": "2026-10-17 17:55:38.226296+00:00"
},
{
"event": "lease_expired",
"details": {},
"created_at": "2026-10-17 17:55:46.421706+00:00"
},
{
"event": "queued",
"details": {
"reason": "worker_timeout",
"attempt": 1,
"delay_seconds": 0.3398410589033632
},
"created_at": "2026-10-17 17:55:46.422465+00:00"
},
{
"event": "published",
"details": {
"attempt": 2,
"stream_id": "1792259746833-0"
},
"created_at": "2026-10-17 17:55:46.839055+00:00"
},
{
"event": "claimed",
"details": {
"pid": 18,
"fence": "7816a4aa-f4aa-4e6b-b3d9-d4ae66b3de79",
"attempt": 2
},
"created_at": "2026-10-17 17:55:47.464227+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:55:47.514568+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:55:47.515976+00:00"
}
],
"terminal_before_replay": null,
"backoff_delays": [
0.3398410589033632
]
},
{
"scenario": "permanent_1_failed_then_explicit_replay",
"attachment_id": "6ccff855-f892-4601-bbf5-2893fae67e8f",
"job_id": "a3f87516-287c-42be-aa97-3bf8b5885d40",
"elapsed_seconds": 6.046831666999424,
"final_state": "ready",
"attempts": 1,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:55:48.846819+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259748878-0"
},
"created_at": "2026-10-17 17:55:48.880069+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "800ba7eb-8498-40c1-932a-f4ea7e395664",
"attempt": 1
},
"created_at": "2026-10-17 17:55:52.428500+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "before_effect",
"action": "permanent"
},
"created_at": "2026-10-17 17:55:52.429868+00:00"
},
{
"event": "failed",
"details": {
"reason": "invalid_image",
"attempt": 1,
"delay_seconds": 1.3147411893417413
},
"created_at": "2026-10-17 17:55:52.431235+00:00"
},
{
"event": "operator_replay",
"details": {},
"created_at": "2026-10-17 17:55:53.126759+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259753159-0"
},
"created_at": "2026-10-17 17:55:53.161185+00:00"
},
{
"event": "claimed",
"details": {
"pid": 23,
"fence": "87290761-06ff-4a8f-b6d5-ad6976f3ed88",
"attempt": 1
},
"created_at": "2026-10-17 17:55:53.793623+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:55:53.844275+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:55:53.845675+00:00"
}
],
"terminal_before_replay": {
"state": "failed",
"attempt": 1,
"error_code": "invalid_image"
},
"backoff_delays": []
},
{
"scenario": "transient_5_failed_then_explicit_replay",
"attachment_id": "a501923a-805f-4368-9080-3f79cd73ead4",
"job_id": "151733d6-47fb-406d-8eb4-c2100b0308e0",
"elapsed_seconds": 21.676316540999323,
"final_state": "ready",
"attempts": 1,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:55:55.185222+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259755189-0"
},
"created_at": "2026-10-17 17:55:55.192313+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "9cebcac0-4ba3-4d2d-b90e-574d554f3767",
"attempt": 1
},
"created_at": "2026-10-17 17:55:58.775748+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:55:58.777152+00:00"
},
{
"event": "queued",
"details": {
"reason": "storage_unavailable",
"attempt": 1,
"delay_seconds": 0.5041968356000014
},
"created_at": "2026-10-17 17:55:58.778381+00:00"
},
{
"event": "published",
"details": {
"attempt": 2,
"stream_id": "1792259759462-0"
},
"created_at": "2026-10-17 17:55:59.467840+00:00"
},
{
"event": "claimed",
"details": {
"pid": 13,
"fence": "16ba040b-3be8-45b2-871b-4304ce87436f",
"attempt": 2
},
"created_at": "2026-10-17 17:56:00.097726+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 13,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:56:00.099153+00:00"
},
{
"event": "queued",
"details": {
"reason": "storage_unavailable",
"attempt": 2,
"delay_seconds": 3.734568616359563
},
"created_at": "2026-10-17 17:56:00.100559+00:00"
},
{
"event": "published",
"details": {
"attempt": 3,
"stream_id": "1792259763946-0"
},
"created_at": "2026-10-17 17:56:03.950595+00:00"
},
{
"event": "claimed",
"details": {
"pid": 18,
"fence": "ab7e8542-165c-41ce-a598-4a9436561994",
"attempt": 3
},
"created_at": "2026-10-17 17:56:04.568035+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 18,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:56:04.569436+00:00"
},
{
"event": "queued",
"details": {
"reason": "storage_unavailable",
"attempt": 3,
"delay_seconds": 6.758513583967568
},
"created_at": "2026-10-17 17:56:04.570875+00:00"
},
{
"event": "published",
"details": {
"attempt": 4,
"stream_id": "1792259771523-0"
},
"created_at": "2026-10-17 17:56:11.529321+00:00"
},
{
"event": "claimed",
"details": {
"pid": 23,
"fence": "da27b3e3-86a6-4db2-9c51-bc4006daeca6",
"attempt": 4
},
"created_at": "2026-10-17 17:56:12.154043+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 23,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:56:12.155207+00:00"
},
{
"event": "queued",
"details": {
"reason": "storage_unavailable",
"attempt": 4,
"delay_seconds": 1.6177495364053143
},
"created_at": "2026-10-17 17:56:12.156436+00:00"
},
{
"event": "published",
"details": {
"attempt": 5,
"stream_id": "1792259773778-0"
},
"created_at": "2026-10-17 17:56:13.783870+00:00"
},
{
"event": "claimed",
"details": {
"pid": 28,
"fence": "cfa76523-617c-4287-a078-58fdc820a47a",
"attempt": 5
},
"created_at": "2026-10-17 17:56:14.403020+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 28,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:56:14.406348+00:00"
},
{
"event": "failed",
"details": {
"reason": "retry_exhausted",
"attempt": 5,
"delay_seconds": 6.39169831883081
},
"created_at": "2026-10-17 17:56:14.407579+00:00"
},
{
"event": "operator_replay",
"details": {},
"created_at": "2026-10-17 17:56:15.093252+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259775208-0"
},
"created_at": "2026-10-17 17:56:15.212412+00:00"
},
{
"event": "claimed",
"details": {
"pid": 43,
"fence": "7155f9fa-382f-42dc-8793-794bd8656281",
"attempt": 1
},
"created_at": "2026-10-17 17:56:15.836518+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:56:15.883836+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:56:15.885318+00:00"
}
],
"terminal_before_replay": {
"state": "failed",
"attempt": 5,
"error_code": "retry_exhausted"
},
"backoff_delays": [
0.5041968356000014,
3.734568616359563,
6.758513583967568,
1.6177495364053143
]
},
{
"scenario": "reorder_stale_v1_after_v2_and_publication_fence",
"attachment_id": "1dead2ee-b162-44c8-a768-8c0e8720da9a",
"job_id": "ce9d85cf-a7e4-4fb8-8d52-3772ea5aac22",
"elapsed_seconds": 19.608258917000057,
"final_state": "ready",
"attempts": 1,
"final_version": 2,
"published_objects": 4,
"audit": [
{
"event": "new_version",
"details": {
"previous_job": "dde5f4e7-0ab7-488b-a195-187d990397b7"
},
"created_at": "2026-10-17 17:56:17.252860+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259778981-0"
},
"created_at": "2026-10-17 17:56:18.992904+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "36c7f8f2-4b08-42be-9c65-561ee1a2f593",
"attempt": 1
},
"created_at": "2026-10-17 17:56:20.912305+00:00"
},
{
"event": "objects_written",
"details": {
"version": 2
},
"created_at": "2026-10-17 17:56:20.964375+00:00"
},
{
"event": "result_committed",
"details": {
"version": 2
},
"created_at": "2026-10-17 17:56:20.965987+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:21.668700+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:22.475318+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:23.240148+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:24.005857+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:24.762654+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:25.544880+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:26.317008+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:27.077893+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:27.861434+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:28.672986+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:29.476919+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:30.254220+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:31.131393+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:31.928877+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:32.753829+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:33.570694+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:34.375167+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:35.137268+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:35.904138+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:56:36.672446+00:00"
}
],
"version": 2,
"old_audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:56:17.238597+00:00"
},
{
"event": "claimed",
"details": {
"pid": 17635,
"fence": "500e09ff-0f9d-4a59-9084-ecb7c0fe2bd0",
"attempt": 1
},
"created_at": "2026-10-17 17:56:17.247016+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:20.909821+00:00"
},
{
"event": "publication_fenced",
"details": {},
"created_at": "2026-10-17 17:56:21.215333+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:21.714782+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:22.476928+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:23.238615+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:24.004298+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:24.821420+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:25.607833+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:26.375490+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:27.127431+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:27.911537+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:28.724080+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:29.478445+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:30.260567+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:31.132939+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:31.927497+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:32.752154+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:33.569033+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:34.373552+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:35.196087+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:36.003234+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:56:36.786981+00:00"
}
],
"duplicates": 44
}
],
"accepted": 11,
"lost_confirmed": 0,
"stale_overwrites": 0,
"worker_cgroup": {
"memory.current": "83656704",
"memory.peak": "218660864",
"memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0",
"cpu.stat": "usage_usec 35889545\nuser_usec 33648322\nsystem_usec 2241223\nnice_usec 0\nnr_periods 213\nnr_throttled 48\nthrottled_usec 221931\nnr_bursts 0\nburst_usec 0"
},
"logs_credentials_absent": true
}