Files
mtusi2026lab5/evidence/pipeline.json
T

1308 lines
36 KiB
JSON

{
"anonymous_s3_status": 403,
"denied_iam_actions": [
"api:put_object",
"api:delete_object",
"worker:delete_object",
"reconciler:put_object"
],
"cross_replica": {
"uploader": "be8e971137cb4314b5224ba716301444",
"stopped": {
"id": "8d4298b6ad5314fcdf16aa51d35dc303f907eb108469036411bfce1bff40dd72",
"started_at": "2026-10-17T17:45:54.007155055Z",
"pid": 0,
"running": false,
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {}
},
"reader": "4d81bcd18cd841d58fb480df271130ec"
},
"track": 2,
"delivery": "at-least-once",
"effect_key": [
"attachment_id",
"pipeline_version",
"variant"
],
"worker_concurrency": 2,
"prefetch_per_slot": 1,
"job_timeout_seconds": 5,
"lease_seconds": 8,
"max_attempts": 5,
"scenarios": [
{
"scenario": "commit_publish_gap_api_restart_broker_sigkill",
"attachment_id": "1e474e25-eca7-477e-aab6-83cff7e30881",
"job_id": "c8560951-e938-4dcd-be24-0e8862c13879",
"elapsed_seconds": 12.721676291999756,
"final_state": "ready",
"attempts": 1,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:45:53.083710+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259161144-0"
},
"created_at": "2026-10-17 17:46:01.152164+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "cbec1efe-f93d-4e7e-ad80-9aca46147323",
"attempt": 1
},
"created_at": "2026-10-17 17:46:05.429495+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:46:05.484833+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:46:05.486327+00:00"
}
],
"api_before": [
{
"id": "8d4298b6ad5314fcdf16aa51d35dc303f907eb108469036411bfce1bff40dd72",
"started_at": "2026-10-17T17:45:01.430679877Z",
"pid": 40417,
"running": true,
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "3eb1f9c99bab4c76af6d2a7c69753685"
},
{
"id": "a59075a90d19de7f3057ad13490e5fba946c1f714e1f101bb8abaee40dd7201e",
"started_at": "2026-10-17T17:45:01.509888461Z",
"pid": 40472,
"running": true,
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "233e879e5ca9468bbde35ef6c58e0278"
}
],
"api_after": [
{
"id": "8d4298b6ad5314fcdf16aa51d35dc303f907eb108469036411bfce1bff40dd72",
"started_at": "2026-10-17T17:45:54.007155055Z",
"pid": 45186,
"running": true,
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "be8e971137cb4314b5224ba716301444"
},
{
"id": "a59075a90d19de7f3057ad13490e5fba946c1f714e1f101bb8abaee40dd7201e",
"started_at": "2026-10-17T17:45:53.909326888Z",
"pid": 45122,
"running": true,
"image": "sha256:691367499a6453eaf54b56c9782f0c8ee7f88b7e63a41c5e686a191353c7d96a",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "4d81bcd18cd841d58fb480df271130ec"
}
],
"broker_before": {
"id": "e21054f56ca9160a2587b123cb21624e2685ebc7e252df354add56c704032f7b",
"started_at": "2026-10-17T17:41:37.030985963Z",
"pid": 20684,
"running": true,
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
"memory": 402653184,
"nano_cpus": 500000000,
"published_ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56385"
}
]
}
},
"broker_after": {
"id": "e21054f56ca9160a2587b123cb21624e2685ebc7e252df354add56c704032f7b",
"started_at": "2026-10-17T17:46:01.597192669Z",
"pid": 46402,
"running": true,
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
"memory": 402653184,
"nano_cpus": 500000000,
"published_ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56385"
}
]
}
},
"broker_entry_ids": [
"1792259161144-0"
],
"message_id": "4337fa6f-8f98-4395-9a76-77c69280b7d7"
},
{
"scenario": "publish_before_outbox_commit_duplicate",
"attachment_id": "40a6392f-7cd8-4454-937a-91110cf3803d",
"job_id": "14af5e44-e36c-4fe6-a41b-c8f8368d2398",
"elapsed_seconds": 14.330121625000174,
"final_state": "ready",
"attempts": 1,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:46:06.202205+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 1,
"phase": "after_publish",
"action": "crash"
},
"created_at": "2026-10-17 17:46:07.954992+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259168643-0"
},
"created_at": "2026-10-17 17:46:08.650794+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "eba8ae13-0eb5-4fbc-a9df-712f56818d5e",
"attempt": 1
},
"created_at": "2026-10-17 17:46:12.346429+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:46:12.399929+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:46:12.401716+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:46:20.291681+00:00"
}
]
},
{
"scenario": "worker_crash_before_effect",
"attachment_id": "12addd02-9a47-4094-92db-877d256ae7d9",
"job_id": "9f1a5c70-7f9f-4dc8-a792-9ffa39c289d6",
"elapsed_seconds": 13.894031958001506,
"final_state": "ready",
"attempts": 2,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:46:20.868173+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259180875-0"
},
"created_at": "2026-10-17 17:46:20.876688+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "60bb41fe-f4ac-4f92-85a3-34f29bfc96c9",
"attempt": 1
},
"created_at": "2026-10-17 17:46:24.419572+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "before_effect",
"action": "crash"
},
"created_at": "2026-10-17 17:46:24.421010+00:00"
},
{
"event": "lease_expired",
"details": {},
"created_at": "2026-10-17 17:46:32.494088+00:00"
},
{
"event": "queued",
"details": {
"reason": "worker_timeout",
"attempt": 1,
"delay_seconds": 1.2896975596214408
},
"created_at": "2026-10-17 17:46:32.494762+00:00"
},
{
"event": "published",
"details": {
"attempt": 2,
"stream_id": "1792259193929-0"
},
"created_at": "2026-10-17 17:46:33.934245+00:00"
},
{
"event": "claimed",
"details": {
"pid": 18,
"fence": "5ce3687f-6595-4929-8275-67b2035127df",
"attempt": 2
},
"created_at": "2026-10-17 17:46:34.557940+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:46:34.604217+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:46:34.605482+00:00"
}
]
},
{
"scenario": "worker_crash_after_effect",
"attachment_id": "11449622-369e-4fb7-8182-3b9b3d0176ec",
"job_id": "869513a3-aa7a-49ef-9c53-b2249e212c8b",
"elapsed_seconds": 13.864409916999648,
"final_state": "ready",
"attempts": 2,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:46:35.079926+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259195147-0"
},
"created_at": "2026-10-17 17:46:35.151605+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "4e357bdf-3eae-45d4-a6c8-abe637120a70",
"attempt": 1
},
"created_at": "2026-10-17 17:46:38.636974+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "after_effect",
"action": "crash"
},
"created_at": "2026-10-17 17:46:38.658057+00:00"
},
{
"event": "lease_expired",
"details": {},
"created_at": "2026-10-17 17:46:46.740953+00:00"
},
{
"event": "queued",
"details": {
"reason": "worker_timeout",
"attempt": 1,
"delay_seconds": 1.410429264025676
},
"created_at": "2026-10-17 17:46:46.741495+00:00"
},
{
"event": "published",
"details": {
"attempt": 2,
"stream_id": "1792259208170-0"
},
"created_at": "2026-10-17 17:46:48.175400+00:00"
},
{
"event": "claimed",
"details": {
"pid": 23,
"fence": "ab6508c7-fd00-488f-87e9-29e0954b1776",
"attempt": 2
},
"created_at": "2026-10-17 17:46:48.789656+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:46:48.836495+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:46:48.837868+00:00"
}
]
},
{
"scenario": "worker_crash_after_commit",
"attachment_id": "f1041b2e-2fe0-47ed-a043-d84886c0df7c",
"job_id": "b866e882-984d-45ba-a698-ba0ce86d9879",
"elapsed_seconds": 11.775217083000825,
"final_state": "ready",
"attempts": 1,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:46:49.197189+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259209396-0"
},
"created_at": "2026-10-17 17:46:49.400324+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "117e2aef-df59-43c0-b9b8-59e917eecc70",
"attempt": 1
},
"created_at": "2026-10-17 17:46:52.753680+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:46:52.800148+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:46:52.801552+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "after_commit",
"action": "crash"
},
"created_at": "2026-10-17 17:46:52.802279+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:47:00.823126+00:00"
}
]
},
{
"scenario": "transient_2",
"attachment_id": "97f571e2-73a0-4f14-93d9-e01002c11f61",
"job_id": "18dc7d60-1f9e-4f61-833c-4949494c8b91",
"elapsed_seconds": 8.707766999999876,
"final_state": "ready",
"attempts": 3,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:47:01.389150+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259221413-0"
},
"created_at": "2026-10-17 17:47:01.415406+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "18997f6d-a01a-4be4-b062-b5a2e8ac483e",
"attempt": 1
},
"created_at": "2026-10-17 17:47:04.944561+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:47:04.945841+00:00"
},
{
"event": "queued",
"details": {
"reason": "storage_unavailable",
"attempt": 1,
"delay_seconds": 1.7045145018758805
},
"created_at": "2026-10-17 17:47:04.946935+00:00"
},
{
"event": "published",
"details": {
"attempt": 2,
"stream_id": "1792259226717-0"
},
"created_at": "2026-10-17 17:47:06.723352+00:00"
},
{
"event": "claimed",
"details": {
"pid": 13,
"fence": "3e942ab1-4fb5-47dc-b286-9e8d2f8c577a",
"attempt": 2
},
"created_at": "2026-10-17 17:47:07.361069+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 13,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:47:07.362398+00:00"
},
{
"event": "queued",
"details": {
"reason": "storage_unavailable",
"attempt": 2,
"delay_seconds": 1.0201311878586972
},
"created_at": "2026-10-17 17:47:07.363567+00:00"
},
{
"event": "published",
"details": {
"attempt": 3,
"stream_id": "1792259228557-0"
},
"created_at": "2026-10-17 17:47:08.563450+00:00"
},
{
"event": "claimed",
"details": {
"pid": 18,
"fence": "3a1bbe6e-88e7-42ad-8a9f-e5feafdf2f68",
"attempt": 3
},
"created_at": "2026-10-17 17:47:09.187031+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:47:09.236868+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:47:09.238352+00:00"
}
],
"terminal_before_replay": null,
"backoff_delays": [
1.7045145018758805,
1.0201311878586972
]
},
{
"scenario": "hang_1",
"attachment_id": "e62fd6c8-685e-48ec-8a3b-ec4a5617186f",
"job_id": "3c2c7c87-b165-4cf5-b955-95a671c85448",
"elapsed_seconds": 13.661484917000053,
"final_state": "ready",
"attempts": 2,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:47:10.344093+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259230384-0"
},
"created_at": "2026-10-17 17:47:10.385872+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "a3b187eb-4890-4617-b324-5991d7f17994",
"attempt": 1
},
"created_at": "2026-10-17 17:47:13.906329+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "before_effect",
"action": "hang"
},
"created_at": "2026-10-17 17:47:13.907760+00:00"
},
{
"event": "lease_expired",
"details": {},
"created_at": "2026-10-17 17:47:22.007914+00:00"
},
{
"event": "queued",
"details": {
"reason": "worker_timeout",
"attempt": 1,
"delay_seconds": 0.25001997265740394
},
"created_at": "2026-10-17 17:47:22.008469+00:00"
},
{
"event": "published",
"details": {
"attempt": 2,
"stream_id": "1792259242414-0"
},
"created_at": "2026-10-17 17:47:22.417567+00:00"
},
{
"event": "claimed",
"details": {
"pid": 18,
"fence": "3c367882-b9db-4949-aba0-ff35d7ce4feb",
"attempt": 2
},
"created_at": "2026-10-17 17:47:23.032980+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:47:23.079047+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:47:23.080474+00:00"
}
],
"terminal_before_replay": null,
"backoff_delays": [
0.25001997265740394
]
},
{
"scenario": "permanent_1_failed_then_explicit_replay",
"attachment_id": "e0f1b3dc-6ab3-4936-a170-e50737ffda23",
"job_id": "f06249e9-4bea-4908-8f13-759cd34fce31",
"elapsed_seconds": 6.0364980829999695,
"final_state": "ready",
"attempts": 1,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:47:24.346756+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259244438-0"
},
"created_at": "2026-10-17 17:47:24.441767+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "910286eb-c3d5-4ffc-aedf-c7b7b303b5b1",
"attempt": 1
},
"created_at": "2026-10-17 17:47:27.913900+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "before_effect",
"action": "permanent"
},
"created_at": "2026-10-17 17:47:27.915304+00:00"
},
{
"event": "failed",
"details": {
"reason": "invalid_image",
"attempt": 1,
"delay_seconds": 1.7202167777743547
},
"created_at": "2026-10-17 17:47:27.916542+00:00"
},
{
"event": "operator_replay",
"details": {},
"created_at": "2026-10-17 17:47:28.624269+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259248694-0"
},
"created_at": "2026-10-17 17:47:28.698041+00:00"
},
{
"event": "claimed",
"details": {
"pid": 23,
"fence": "f9ef36fb-53ba-41d3-acfd-345227b5eae3",
"attempt": 1
},
"created_at": "2026-10-17 17:47:29.328542+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:47:29.376012+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:47:29.377432+00:00"
}
],
"terminal_before_replay": {
"state": "failed",
"attempt": 1,
"error_code": "invalid_image"
},
"backoff_delays": []
},
{
"scenario": "transient_5_failed_then_explicit_replay",
"attachment_id": "c1112da4-0e83-4f2a-bf50-1b9d539e5d1d",
"job_id": "5fe52a90-f979-4142-ba5a-0fde2df1b999",
"elapsed_seconds": 30.797973708000427,
"final_state": "ready",
"attempts": 1,
"final_version": 1,
"published_objects": 4,
"audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:47:30.769224+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259250927-0"
},
"created_at": "2026-10-17 17:47:30.928851+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "610513e2-9127-47be-bcf3-763e203b9270",
"attempt": 1
},
"created_at": "2026-10-17 17:47:34.326981+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 8,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:47:34.328316+00:00"
},
{
"event": "queued",
"details": {
"reason": "storage_unavailable",
"attempt": 1,
"delay_seconds": 0.8984755498946879
},
"created_at": "2026-10-17 17:47:34.329569+00:00"
},
{
"event": "published",
"details": {
"attempt": 2,
"stream_id": "1792259255393-0"
},
"created_at": "2026-10-17 17:47:35.397234+00:00"
},
{
"event": "claimed",
"details": {
"pid": 13,
"fence": "2ae77e2a-db7b-45a8-955d-7e20d1c0dbc4",
"attempt": 2
},
"created_at": "2026-10-17 17:47:36.019890+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 13,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:47:36.021306+00:00"
},
{
"event": "queued",
"details": {
"reason": "storage_unavailable",
"attempt": 2,
"delay_seconds": 2.329669916187136
},
"created_at": "2026-10-17 17:47:36.022525+00:00"
},
{
"event": "published",
"details": {
"attempt": 3,
"stream_id": "1792259258449-0"
},
"created_at": "2026-10-17 17:47:38.458491+00:00"
},
{
"event": "claimed",
"details": {
"pid": 18,
"fence": "060bdca5-353a-466f-811a-4ffd540392c1",
"attempt": 3
},
"created_at": "2026-10-17 17:47:39.080868+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 18,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:47:39.082167+00:00"
},
{
"event": "queued",
"details": {
"reason": "storage_unavailable",
"attempt": 3,
"delay_seconds": 5.378865191423077
},
"created_at": "2026-10-17 17:47:39.083325+00:00"
},
{
"event": "published",
"details": {
"attempt": 4,
"stream_id": "1792259264591-0"
},
"created_at": "2026-10-17 17:47:44.597031+00:00"
},
{
"event": "claimed",
"details": {
"pid": 23,
"fence": "405e41de-268d-4054-9e1e-bb195e9de674",
"attempt": 4
},
"created_at": "2026-10-17 17:47:45.213122+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 23,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:47:45.214460+00:00"
},
{
"event": "queued",
"details": {
"reason": "storage_unavailable",
"attempt": 4,
"delay_seconds": 13.418589126321523
},
"created_at": "2026-10-17 17:47:45.215675+00:00"
},
{
"event": "published",
"details": {
"attempt": 5,
"stream_id": "1792259278692-0"
},
"created_at": "2026-10-17 17:47:58.698162+00:00"
},
{
"event": "claimed",
"details": {
"pid": 28,
"fence": "1faf9428-20ba-4378-a69c-bbedb0b68e99",
"attempt": 5
},
"created_at": "2026-10-17 17:47:59.329661+00:00"
},
{
"event": "fault_injected",
"details": {
"pid": 28,
"phase": "before_effect",
"action": "transient"
},
"created_at": "2026-10-17 17:47:59.331004+00:00"
},
{
"event": "failed",
"details": {
"reason": "retry_exhausted",
"attempt": 5,
"delay_seconds": 7.662262510573656
},
"created_at": "2026-10-17 17:47:59.332436+00:00"
},
{
"event": "operator_replay",
"details": {},
"created_at": "2026-10-17 17:48:00.108607+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259280116-0"
},
"created_at": "2026-10-17 17:48:00.118492+00:00"
},
{
"event": "claimed",
"details": {
"pid": 43,
"fence": "71fa83a8-b49e-4512-87fa-6e647bf596c6",
"attempt": 1
},
"created_at": "2026-10-17 17:48:00.733111+00:00"
},
{
"event": "objects_written",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:48:00.780352+00:00"
},
{
"event": "result_committed",
"details": {
"version": 1
},
"created_at": "2026-10-17 17:48:00.781774+00:00"
}
],
"terminal_before_replay": {
"state": "failed",
"attempt": 5,
"error_code": "retry_exhausted"
},
"backoff_delays": [
0.8984755498946879,
2.329669916187136,
5.378865191423077,
13.418589126321523
]
},
{
"scenario": "reorder_stale_v1_after_v2_and_publication_fence",
"attachment_id": "068fc0f6-01af-4315-afd7-7f4d6d0b731a",
"job_id": "b9f6bb9d-2d47-4dcc-bdc2-a542dd4e3927",
"elapsed_seconds": 19.354204458999448,
"final_state": "ready",
"attempts": 1,
"final_version": 2,
"published_objects": 4,
"audit": [
{
"event": "new_version",
"details": {
"previous_job": "4d640238-d3ed-4185-8d2e-174373de5ce2"
},
"created_at": "2026-10-17 17:48:02.024786+00:00"
},
{
"event": "published",
"details": {
"attempt": 1,
"stream_id": "1792259283732-0"
},
"created_at": "2026-10-17 17:48:03.744770+00:00"
},
{
"event": "claimed",
"details": {
"pid": 8,
"fence": "b9689bf7-56f6-42df-823c-479db6818491",
"attempt": 1
},
"created_at": "2026-10-17 17:48:05.640199+00:00"
},
{
"event": "objects_written",
"details": {
"version": 2
},
"created_at": "2026-10-17 17:48:05.691195+00:00"
},
{
"event": "result_committed",
"details": {
"version": 2
},
"created_at": "2026-10-17 17:48:05.692582+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:06.410645+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:07.171456+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:07.939514+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:08.706418+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:09.477508+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:10.248560+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:11.017774+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:11.784105+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:12.556187+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:13.330030+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:14.099539+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:14.850770+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:15.623106+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:16.437431+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:17.214748+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:18.021099+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:18.779828+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:19.555733+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:20.367879+00:00"
},
{
"event": "duplicate_ready",
"details": {},
"created_at": "2026-10-17 17:48:21.139160+00:00"
}
],
"version": 2,
"old_audit": [
{
"event": "accepted",
"details": {},
"created_at": "2026-10-17 17:48:02.009745+00:00"
},
{
"event": "claimed",
"details": {
"pid": 16485,
"fence": "5fc23a49-4350-4589-beac-182a8441da90",
"attempt": 1
},
"created_at": "2026-10-17 17:48:02.018367+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:05.641554+00:00"
},
{
"event": "publication_fenced",
"details": {},
"created_at": "2026-10-17 17:48:05.962024+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:06.467394+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:07.224257+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:07.995435+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:08.764814+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:09.534368+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:10.300571+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:11.073402+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:11.846936+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:12.613547+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:13.380369+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:14.141996+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:14.911247+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:15.676589+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:16.438979+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:17.213111+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:18.022730+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:18.781314+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:19.559624+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:20.369480+00:00"
},
{
"event": "stale_delivery",
"details": {},
"created_at": "2026-10-17 17:48:21.137638+00:00"
}
],
"duplicates": 44
}
],
"accepted": 11,
"lost_confirmed": 0,
"stale_overwrites": 0,
"worker_cgroup": {
"memory.current": "83546112",
"memory.peak": "219336704",
"memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0",
"cpu.stat": "usage_usec 35101943\nuser_usec 32958642\nsystem_usec 2143301\nnice_usec 0\nnr_periods 203\nnr_throttled 38\nthrottled_usec 199981\nnr_bursts 0\nburst_usec 0"
},
"logs_credentials_absent": true
}