forked from mtusi2026/mtusi2026lab5
1308 lines
36 KiB
JSON
1308 lines
36 KiB
JSON
{
|
|
"anonymous_s3_status": 403,
|
|
"denied_iam_actions": [
|
|
"api:put_object",
|
|
"api:delete_object",
|
|
"worker:delete_object",
|
|
"reconciler:put_object"
|
|
],
|
|
"cross_replica": {
|
|
"uploader": "0a9437289ec643a2bfc1fe79c1409f2b",
|
|
"stopped": {
|
|
"id": "1cf780371b6e3d5a10b50b91dd1dadddb1371f7f1d2c94bbc0437adc45ae1f4d",
|
|
"started_at": "2026-10-17T17:54:22.316986762Z",
|
|
"pid": 0,
|
|
"running": false,
|
|
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1",
|
|
"memory": 536870912,
|
|
"nano_cpus": 1000000000,
|
|
"published_ports": {}
|
|
},
|
|
"reader": "52ae54c070824ef39ea11614a3a3684a"
|
|
},
|
|
"track": 2,
|
|
"delivery": "at-least-once",
|
|
"effect_key": [
|
|
"attachment_id",
|
|
"pipeline_version",
|
|
"variant"
|
|
],
|
|
"worker_concurrency": 2,
|
|
"prefetch_per_slot": 1,
|
|
"job_timeout_seconds": 5,
|
|
"lease_seconds": 8,
|
|
"max_attempts": 5,
|
|
"scenarios": [
|
|
{
|
|
"scenario": "commit_publish_gap_api_restart_broker_sigkill",
|
|
"attachment_id": "bffcca56-87f6-4e5f-9f0b-b246e3216f13",
|
|
"job_id": "07853ae0-ebf0-4f59-a835-c6c47bbec488",
|
|
"elapsed_seconds": 10.150205792000634,
|
|
"final_state": "ready",
|
|
"attempts": 1,
|
|
"final_version": 1,
|
|
"published_objects": 4,
|
|
"audit": [
|
|
{
|
|
"event": "accepted",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:54:21.522737+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 1,
|
|
"stream_id": "1792259667046-0"
|
|
},
|
|
"created_at": "2026-10-17 17:54:27.056252+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 8,
|
|
"fence": "00c89838-3f0d-429b-9641-ffd458180ba1",
|
|
"attempt": 1
|
|
},
|
|
"created_at": "2026-10-17 17:54:31.193598+00:00"
|
|
},
|
|
{
|
|
"event": "objects_written",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:54:31.240139+00:00"
|
|
},
|
|
{
|
|
"event": "result_committed",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:54:31.241485+00:00"
|
|
}
|
|
],
|
|
"api_before": [
|
|
{
|
|
"id": "50f1efb1b58d309cf2f9084d2af4270dbdf26f11b211cee05b02b3d555e1c250",
|
|
"started_at": "2026-10-17T17:53:40.323395673Z",
|
|
"pid": 82578,
|
|
"running": true,
|
|
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1",
|
|
"memory": 536870912,
|
|
"nano_cpus": 1000000000,
|
|
"published_ports": {},
|
|
"instance": "2df8aa848dbe43f7b28d10fb379202f8"
|
|
},
|
|
{
|
|
"id": "1cf780371b6e3d5a10b50b91dd1dadddb1371f7f1d2c94bbc0437adc45ae1f4d",
|
|
"started_at": "2026-10-17T17:53:40.39698984Z",
|
|
"pid": 82634,
|
|
"running": true,
|
|
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1",
|
|
"memory": 536870912,
|
|
"nano_cpus": 1000000000,
|
|
"published_ports": {},
|
|
"instance": "54220b0dbefb48bfbc470fbf31a6bd96"
|
|
}
|
|
],
|
|
"api_after": [
|
|
{
|
|
"id": "50f1efb1b58d309cf2f9084d2af4270dbdf26f11b211cee05b02b3d555e1c250",
|
|
"started_at": "2026-10-17T17:54:22.360219637Z",
|
|
"pid": 86502,
|
|
"running": true,
|
|
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1",
|
|
"memory": 536870912,
|
|
"nano_cpus": 1000000000,
|
|
"published_ports": {},
|
|
"instance": "52ae54c070824ef39ea11614a3a3684a"
|
|
},
|
|
{
|
|
"id": "1cf780371b6e3d5a10b50b91dd1dadddb1371f7f1d2c94bbc0437adc45ae1f4d",
|
|
"started_at": "2026-10-17T17:54:22.316986762Z",
|
|
"pid": 86447,
|
|
"running": true,
|
|
"image": "sha256:b808cbdf44dcd1bff4a0361e5d7da3786f9fe0937f24053bb0df3b7112d8b6d1",
|
|
"memory": 536870912,
|
|
"nano_cpus": 1000000000,
|
|
"published_ports": {},
|
|
"instance": "0a9437289ec643a2bfc1fe79c1409f2b"
|
|
}
|
|
],
|
|
"broker_before": {
|
|
"id": "95c5c9f8fb2d909cada1fa7e3b45f81524d42b83b72b7dba1029b8bf705f3b25",
|
|
"started_at": "2026-10-17T17:53:35.222258712Z",
|
|
"pid": 81855,
|
|
"running": true,
|
|
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
|
|
"memory": 402653184,
|
|
"nano_cpus": 500000000,
|
|
"published_ports": {
|
|
"6379/tcp": [
|
|
{
|
|
"HostIp": "127.0.0.1",
|
|
"HostPort": "56385"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"broker_after": {
|
|
"id": "95c5c9f8fb2d909cada1fa7e3b45f81524d42b83b72b7dba1029b8bf705f3b25",
|
|
"started_at": "2026-10-17T17:54:27.551506417Z",
|
|
"pid": 87340,
|
|
"running": true,
|
|
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
|
|
"memory": 402653184,
|
|
"nano_cpus": 500000000,
|
|
"published_ports": {
|
|
"6379/tcp": [
|
|
{
|
|
"HostIp": "127.0.0.1",
|
|
"HostPort": "56385"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"broker_entry_ids": [
|
|
"1792259667046-0"
|
|
],
|
|
"message_id": "d121d1ff-ac42-48c1-ba36-6ef3fa377015"
|
|
},
|
|
{
|
|
"scenario": "publish_before_outbox_commit_duplicate",
|
|
"attachment_id": "ccf6515f-d80d-40db-be0e-33cc21684fe9",
|
|
"job_id": "47da08ce-cbc3-4ef0-abc6-f3275a7f3ef2",
|
|
"elapsed_seconds": 14.278877541999464,
|
|
"final_state": "ready",
|
|
"attempts": 1,
|
|
"final_version": 1,
|
|
"published_objects": 4,
|
|
"audit": [
|
|
{
|
|
"event": "accepted",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:54:32.071860+00:00"
|
|
},
|
|
{
|
|
"event": "fault_injected",
|
|
"details": {
|
|
"pid": 1,
|
|
"phase": "after_publish",
|
|
"action": "crash"
|
|
},
|
|
"created_at": "2026-10-17 17:54:33.771814+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 1,
|
|
"stream_id": "1792259674388-0"
|
|
},
|
|
"created_at": "2026-10-17 17:54:34.396026+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 8,
|
|
"fence": "2b004cdb-3083-4ff8-94c9-7374a104896c",
|
|
"attempt": 1
|
|
},
|
|
"created_at": "2026-10-17 17:54:38.179597+00:00"
|
|
},
|
|
{
|
|
"event": "objects_written",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:54:38.233346+00:00"
|
|
},
|
|
{
|
|
"event": "result_committed",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:54:38.234856+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:54:46.175653+00:00"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"scenario": "worker_crash_before_effect",
|
|
"attachment_id": "36ce0968-4044-4da3-b581-93f4843bc8d6",
|
|
"job_id": "3f333283-15ed-44a9-bd1a-2be5ae212c5e",
|
|
"elapsed_seconds": 13.600872499999241,
|
|
"final_state": "ready",
|
|
"attempts": 2,
|
|
"final_version": 1,
|
|
"published_objects": 4,
|
|
"audit": [
|
|
{
|
|
"event": "accepted",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:54:46.619877+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 1,
|
|
"stream_id": "1792259686809-0"
|
|
},
|
|
"created_at": "2026-10-17 17:54:46.813863+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 8,
|
|
"fence": "795b98c1-968d-4c19-ac55-7e1e15e20fe6",
|
|
"attempt": 1
|
|
},
|
|
"created_at": "2026-10-17 17:54:50.189662+00:00"
|
|
},
|
|
{
|
|
"event": "fault_injected",
|
|
"details": {
|
|
"pid": 8,
|
|
"phase": "before_effect",
|
|
"action": "crash"
|
|
},
|
|
"created_at": "2026-10-17 17:54:50.191055+00:00"
|
|
},
|
|
{
|
|
"event": "lease_expired",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:54:58.255525+00:00"
|
|
},
|
|
{
|
|
"event": "queued",
|
|
"details": {
|
|
"reason": "worker_timeout",
|
|
"attempt": 1,
|
|
"delay_seconds": 1.1620223936252645
|
|
},
|
|
"created_at": "2026-10-17 17:54:58.256553+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 2,
|
|
"stream_id": "1792259699473-0"
|
|
},
|
|
"created_at": "2026-10-17 17:54:59.478214+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 18,
|
|
"fence": "8e2780d0-cf82-4fdc-be8c-9b553794686f",
|
|
"attempt": 2
|
|
},
|
|
"created_at": "2026-10-17 17:55:00.113366+00:00"
|
|
},
|
|
{
|
|
"event": "objects_written",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:00.161563+00:00"
|
|
},
|
|
{
|
|
"event": "result_committed",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:00.163145+00:00"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"scenario": "worker_crash_after_effect",
|
|
"attachment_id": "0b374955-a72f-49fa-ba6c-873174a2a445",
|
|
"job_id": "4feb58df-db81-47f0-b0e7-0fd99256c8b8",
|
|
"elapsed_seconds": 13.586232916999506,
|
|
"final_state": "ready",
|
|
"attempts": 2,
|
|
"final_version": 1,
|
|
"published_objects": 4,
|
|
"audit": [
|
|
{
|
|
"event": "accepted",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:55:00.717466+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 1,
|
|
"stream_id": "1792259700904-0"
|
|
},
|
|
"created_at": "2026-10-17 17:55:00.905775+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 8,
|
|
"fence": "8a11cf54-ccea-4268-8ee4-00ca0bce64c8",
|
|
"attempt": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:04.321489+00:00"
|
|
},
|
|
{
|
|
"event": "fault_injected",
|
|
"details": {
|
|
"pid": 8,
|
|
"phase": "after_effect",
|
|
"action": "crash"
|
|
},
|
|
"created_at": "2026-10-17 17:55:04.350788+00:00"
|
|
},
|
|
{
|
|
"event": "lease_expired",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:55:12.366261+00:00"
|
|
},
|
|
{
|
|
"event": "queued",
|
|
"details": {
|
|
"reason": "worker_timeout",
|
|
"attempt": 1,
|
|
"delay_seconds": 0.8627423719909547
|
|
},
|
|
"created_at": "2026-10-17 17:55:12.366899+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 2,
|
|
"stream_id": "1792259713389-0"
|
|
},
|
|
"created_at": "2026-10-17 17:55:13.395682+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 23,
|
|
"fence": "8db5568d-9e34-4efc-a3b3-7138c347b386",
|
|
"attempt": 2
|
|
},
|
|
"created_at": "2026-10-17 17:55:14.015951+00:00"
|
|
},
|
|
{
|
|
"event": "objects_written",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:14.064324+00:00"
|
|
},
|
|
{
|
|
"event": "result_committed",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:14.065699+00:00"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"scenario": "worker_crash_after_commit",
|
|
"attachment_id": "5ab078f1-6c88-4804-b2eb-020ad667ed44",
|
|
"job_id": "898833da-cd51-4943-8663-4580448c1984",
|
|
"elapsed_seconds": 11.788440333000835,
|
|
"final_state": "ready",
|
|
"attempts": 1,
|
|
"final_version": 1,
|
|
"published_objects": 4,
|
|
"audit": [
|
|
{
|
|
"event": "accepted",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:55:14.613047+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 1,
|
|
"stream_id": "1792259714618-0"
|
|
},
|
|
"created_at": "2026-10-17 17:55:14.620698+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 8,
|
|
"fence": "3ad27bd3-8c04-477e-be69-8319cf9e703d",
|
|
"attempt": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:18.200056+00:00"
|
|
},
|
|
{
|
|
"event": "objects_written",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:18.246743+00:00"
|
|
},
|
|
{
|
|
"event": "result_committed",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:18.248223+00:00"
|
|
},
|
|
{
|
|
"event": "fault_injected",
|
|
"details": {
|
|
"pid": 8,
|
|
"phase": "after_commit",
|
|
"action": "crash"
|
|
},
|
|
"created_at": "2026-10-17 17:55:18.248930+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:55:26.249978+00:00"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"scenario": "transient_2",
|
|
"attachment_id": "e4302e45-e345-46ac-aab4-ac4f6292aaf2",
|
|
"job_id": "e1671b9f-9177-4ba2-9461-a1bef05942d6",
|
|
"elapsed_seconds": 7.436319458000071,
|
|
"final_state": "ready",
|
|
"attempts": 3,
|
|
"final_version": 1,
|
|
"published_objects": 4,
|
|
"audit": [
|
|
{
|
|
"event": "accepted",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:55:26.814495+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 1,
|
|
"stream_id": "1792259726857-0"
|
|
},
|
|
"created_at": "2026-10-17 17:55:26.859168+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 8,
|
|
"fence": "1d7dadee-bc81-46f8-ada0-53600673b332",
|
|
"attempt": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:30.364647+00:00"
|
|
},
|
|
{
|
|
"event": "fault_injected",
|
|
"details": {
|
|
"pid": 8,
|
|
"phase": "before_effect",
|
|
"action": "transient"
|
|
},
|
|
"created_at": "2026-10-17 17:55:30.366081+00:00"
|
|
},
|
|
{
|
|
"event": "queued",
|
|
"details": {
|
|
"reason": "storage_unavailable",
|
|
"attempt": 1,
|
|
"delay_seconds": 0.41945831889983076
|
|
},
|
|
"created_at": "2026-10-17 17:55:30.367279+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 2,
|
|
"stream_id": "1792259730919-0"
|
|
},
|
|
"created_at": "2026-10-17 17:55:30.921761+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 13,
|
|
"fence": "53f36327-c72f-4d27-ad8c-db27d561ba3e",
|
|
"attempt": 2
|
|
},
|
|
"created_at": "2026-10-17 17:55:31.556720+00:00"
|
|
},
|
|
{
|
|
"event": "fault_injected",
|
|
"details": {
|
|
"pid": 13,
|
|
"phase": "before_effect",
|
|
"action": "transient"
|
|
},
|
|
"created_at": "2026-10-17 17:55:31.557980+00:00"
|
|
},
|
|
{
|
|
"event": "queued",
|
|
"details": {
|
|
"reason": "storage_unavailable",
|
|
"attempt": 2,
|
|
"delay_seconds": 0.988456979754063
|
|
},
|
|
"created_at": "2026-10-17 17:55:31.559262+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 3,
|
|
"stream_id": "1792259732564-0"
|
|
},
|
|
"created_at": "2026-10-17 17:55:32.570956+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 18,
|
|
"fence": "af9be6c6-dd49-4d38-8ae9-e145c1d389e7",
|
|
"attempt": 3
|
|
},
|
|
"created_at": "2026-10-17 17:55:33.193577+00:00"
|
|
},
|
|
{
|
|
"event": "objects_written",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:33.245406+00:00"
|
|
},
|
|
{
|
|
"event": "result_committed",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:33.246917+00:00"
|
|
}
|
|
],
|
|
"terminal_before_replay": null,
|
|
"backoff_delays": [
|
|
0.41945831889983076,
|
|
0.988456979754063
|
|
]
|
|
},
|
|
{
|
|
"scenario": "hang_1",
|
|
"attachment_id": "ac601ccb-0c28-4bdc-b3ce-b8b4ad08d8a7",
|
|
"job_id": "cdb07fbb-9ff9-45e5-b46f-39ce1159927a",
|
|
"elapsed_seconds": 13.753177582999342,
|
|
"final_state": "ready",
|
|
"attempts": 2,
|
|
"final_version": 1,
|
|
"published_objects": 4,
|
|
"audit": [
|
|
{
|
|
"event": "accepted",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:55:34.590420+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 1,
|
|
"stream_id": "1792259734606-0"
|
|
},
|
|
"created_at": "2026-10-17 17:55:34.608576+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 8,
|
|
"fence": "42ca70c4-57e4-4fbe-a86c-8ccd83c21749",
|
|
"attempt": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:38.224796+00:00"
|
|
},
|
|
{
|
|
"event": "fault_injected",
|
|
"details": {
|
|
"pid": 8,
|
|
"phase": "before_effect",
|
|
"action": "hang"
|
|
},
|
|
"created_at": "2026-10-17 17:55:38.226296+00:00"
|
|
},
|
|
{
|
|
"event": "lease_expired",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:55:46.421706+00:00"
|
|
},
|
|
{
|
|
"event": "queued",
|
|
"details": {
|
|
"reason": "worker_timeout",
|
|
"attempt": 1,
|
|
"delay_seconds": 0.3398410589033632
|
|
},
|
|
"created_at": "2026-10-17 17:55:46.422465+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 2,
|
|
"stream_id": "1792259746833-0"
|
|
},
|
|
"created_at": "2026-10-17 17:55:46.839055+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 18,
|
|
"fence": "7816a4aa-f4aa-4e6b-b3d9-d4ae66b3de79",
|
|
"attempt": 2
|
|
},
|
|
"created_at": "2026-10-17 17:55:47.464227+00:00"
|
|
},
|
|
{
|
|
"event": "objects_written",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:47.514568+00:00"
|
|
},
|
|
{
|
|
"event": "result_committed",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:47.515976+00:00"
|
|
}
|
|
],
|
|
"terminal_before_replay": null,
|
|
"backoff_delays": [
|
|
0.3398410589033632
|
|
]
|
|
},
|
|
{
|
|
"scenario": "permanent_1_failed_then_explicit_replay",
|
|
"attachment_id": "6ccff855-f892-4601-bbf5-2893fae67e8f",
|
|
"job_id": "a3f87516-287c-42be-aa97-3bf8b5885d40",
|
|
"elapsed_seconds": 6.046831666999424,
|
|
"final_state": "ready",
|
|
"attempts": 1,
|
|
"final_version": 1,
|
|
"published_objects": 4,
|
|
"audit": [
|
|
{
|
|
"event": "accepted",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:55:48.846819+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 1,
|
|
"stream_id": "1792259748878-0"
|
|
},
|
|
"created_at": "2026-10-17 17:55:48.880069+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 8,
|
|
"fence": "800ba7eb-8498-40c1-932a-f4ea7e395664",
|
|
"attempt": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:52.428500+00:00"
|
|
},
|
|
{
|
|
"event": "fault_injected",
|
|
"details": {
|
|
"pid": 8,
|
|
"phase": "before_effect",
|
|
"action": "permanent"
|
|
},
|
|
"created_at": "2026-10-17 17:55:52.429868+00:00"
|
|
},
|
|
{
|
|
"event": "failed",
|
|
"details": {
|
|
"reason": "invalid_image",
|
|
"attempt": 1,
|
|
"delay_seconds": 1.3147411893417413
|
|
},
|
|
"created_at": "2026-10-17 17:55:52.431235+00:00"
|
|
},
|
|
{
|
|
"event": "operator_replay",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:55:53.126759+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 1,
|
|
"stream_id": "1792259753159-0"
|
|
},
|
|
"created_at": "2026-10-17 17:55:53.161185+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 23,
|
|
"fence": "87290761-06ff-4a8f-b6d5-ad6976f3ed88",
|
|
"attempt": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:53.793623+00:00"
|
|
},
|
|
{
|
|
"event": "objects_written",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:53.844275+00:00"
|
|
},
|
|
{
|
|
"event": "result_committed",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:53.845675+00:00"
|
|
}
|
|
],
|
|
"terminal_before_replay": {
|
|
"state": "failed",
|
|
"attempt": 1,
|
|
"error_code": "invalid_image"
|
|
},
|
|
"backoff_delays": []
|
|
},
|
|
{
|
|
"scenario": "transient_5_failed_then_explicit_replay",
|
|
"attachment_id": "a501923a-805f-4368-9080-3f79cd73ead4",
|
|
"job_id": "151733d6-47fb-406d-8eb4-c2100b0308e0",
|
|
"elapsed_seconds": 21.676316540999323,
|
|
"final_state": "ready",
|
|
"attempts": 1,
|
|
"final_version": 1,
|
|
"published_objects": 4,
|
|
"audit": [
|
|
{
|
|
"event": "accepted",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:55:55.185222+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 1,
|
|
"stream_id": "1792259755189-0"
|
|
},
|
|
"created_at": "2026-10-17 17:55:55.192313+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 8,
|
|
"fence": "9cebcac0-4ba3-4d2d-b90e-574d554f3767",
|
|
"attempt": 1
|
|
},
|
|
"created_at": "2026-10-17 17:55:58.775748+00:00"
|
|
},
|
|
{
|
|
"event": "fault_injected",
|
|
"details": {
|
|
"pid": 8,
|
|
"phase": "before_effect",
|
|
"action": "transient"
|
|
},
|
|
"created_at": "2026-10-17 17:55:58.777152+00:00"
|
|
},
|
|
{
|
|
"event": "queued",
|
|
"details": {
|
|
"reason": "storage_unavailable",
|
|
"attempt": 1,
|
|
"delay_seconds": 0.5041968356000014
|
|
},
|
|
"created_at": "2026-10-17 17:55:58.778381+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 2,
|
|
"stream_id": "1792259759462-0"
|
|
},
|
|
"created_at": "2026-10-17 17:55:59.467840+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 13,
|
|
"fence": "16ba040b-3be8-45b2-871b-4304ce87436f",
|
|
"attempt": 2
|
|
},
|
|
"created_at": "2026-10-17 17:56:00.097726+00:00"
|
|
},
|
|
{
|
|
"event": "fault_injected",
|
|
"details": {
|
|
"pid": 13,
|
|
"phase": "before_effect",
|
|
"action": "transient"
|
|
},
|
|
"created_at": "2026-10-17 17:56:00.099153+00:00"
|
|
},
|
|
{
|
|
"event": "queued",
|
|
"details": {
|
|
"reason": "storage_unavailable",
|
|
"attempt": 2,
|
|
"delay_seconds": 3.734568616359563
|
|
},
|
|
"created_at": "2026-10-17 17:56:00.100559+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 3,
|
|
"stream_id": "1792259763946-0"
|
|
},
|
|
"created_at": "2026-10-17 17:56:03.950595+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 18,
|
|
"fence": "ab7e8542-165c-41ce-a598-4a9436561994",
|
|
"attempt": 3
|
|
},
|
|
"created_at": "2026-10-17 17:56:04.568035+00:00"
|
|
},
|
|
{
|
|
"event": "fault_injected",
|
|
"details": {
|
|
"pid": 18,
|
|
"phase": "before_effect",
|
|
"action": "transient"
|
|
},
|
|
"created_at": "2026-10-17 17:56:04.569436+00:00"
|
|
},
|
|
{
|
|
"event": "queued",
|
|
"details": {
|
|
"reason": "storage_unavailable",
|
|
"attempt": 3,
|
|
"delay_seconds": 6.758513583967568
|
|
},
|
|
"created_at": "2026-10-17 17:56:04.570875+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 4,
|
|
"stream_id": "1792259771523-0"
|
|
},
|
|
"created_at": "2026-10-17 17:56:11.529321+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 23,
|
|
"fence": "da27b3e3-86a6-4db2-9c51-bc4006daeca6",
|
|
"attempt": 4
|
|
},
|
|
"created_at": "2026-10-17 17:56:12.154043+00:00"
|
|
},
|
|
{
|
|
"event": "fault_injected",
|
|
"details": {
|
|
"pid": 23,
|
|
"phase": "before_effect",
|
|
"action": "transient"
|
|
},
|
|
"created_at": "2026-10-17 17:56:12.155207+00:00"
|
|
},
|
|
{
|
|
"event": "queued",
|
|
"details": {
|
|
"reason": "storage_unavailable",
|
|
"attempt": 4,
|
|
"delay_seconds": 1.6177495364053143
|
|
},
|
|
"created_at": "2026-10-17 17:56:12.156436+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 5,
|
|
"stream_id": "1792259773778-0"
|
|
},
|
|
"created_at": "2026-10-17 17:56:13.783870+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 28,
|
|
"fence": "cfa76523-617c-4287-a078-58fdc820a47a",
|
|
"attempt": 5
|
|
},
|
|
"created_at": "2026-10-17 17:56:14.403020+00:00"
|
|
},
|
|
{
|
|
"event": "fault_injected",
|
|
"details": {
|
|
"pid": 28,
|
|
"phase": "before_effect",
|
|
"action": "transient"
|
|
},
|
|
"created_at": "2026-10-17 17:56:14.406348+00:00"
|
|
},
|
|
{
|
|
"event": "failed",
|
|
"details": {
|
|
"reason": "retry_exhausted",
|
|
"attempt": 5,
|
|
"delay_seconds": 6.39169831883081
|
|
},
|
|
"created_at": "2026-10-17 17:56:14.407579+00:00"
|
|
},
|
|
{
|
|
"event": "operator_replay",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:15.093252+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 1,
|
|
"stream_id": "1792259775208-0"
|
|
},
|
|
"created_at": "2026-10-17 17:56:15.212412+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 43,
|
|
"fence": "7155f9fa-382f-42dc-8793-794bd8656281",
|
|
"attempt": 1
|
|
},
|
|
"created_at": "2026-10-17 17:56:15.836518+00:00"
|
|
},
|
|
{
|
|
"event": "objects_written",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:56:15.883836+00:00"
|
|
},
|
|
{
|
|
"event": "result_committed",
|
|
"details": {
|
|
"version": 1
|
|
},
|
|
"created_at": "2026-10-17 17:56:15.885318+00:00"
|
|
}
|
|
],
|
|
"terminal_before_replay": {
|
|
"state": "failed",
|
|
"attempt": 5,
|
|
"error_code": "retry_exhausted"
|
|
},
|
|
"backoff_delays": [
|
|
0.5041968356000014,
|
|
3.734568616359563,
|
|
6.758513583967568,
|
|
1.6177495364053143
|
|
]
|
|
},
|
|
{
|
|
"scenario": "reorder_stale_v1_after_v2_and_publication_fence",
|
|
"attachment_id": "1dead2ee-b162-44c8-a768-8c0e8720da9a",
|
|
"job_id": "ce9d85cf-a7e4-4fb8-8d52-3772ea5aac22",
|
|
"elapsed_seconds": 19.608258917000057,
|
|
"final_state": "ready",
|
|
"attempts": 1,
|
|
"final_version": 2,
|
|
"published_objects": 4,
|
|
"audit": [
|
|
{
|
|
"event": "new_version",
|
|
"details": {
|
|
"previous_job": "dde5f4e7-0ab7-488b-a195-187d990397b7"
|
|
},
|
|
"created_at": "2026-10-17 17:56:17.252860+00:00"
|
|
},
|
|
{
|
|
"event": "published",
|
|
"details": {
|
|
"attempt": 1,
|
|
"stream_id": "1792259778981-0"
|
|
},
|
|
"created_at": "2026-10-17 17:56:18.992904+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 8,
|
|
"fence": "36c7f8f2-4b08-42be-9c65-561ee1a2f593",
|
|
"attempt": 1
|
|
},
|
|
"created_at": "2026-10-17 17:56:20.912305+00:00"
|
|
},
|
|
{
|
|
"event": "objects_written",
|
|
"details": {
|
|
"version": 2
|
|
},
|
|
"created_at": "2026-10-17 17:56:20.964375+00:00"
|
|
},
|
|
{
|
|
"event": "result_committed",
|
|
"details": {
|
|
"version": 2
|
|
},
|
|
"created_at": "2026-10-17 17:56:20.965987+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:21.668700+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:22.475318+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:23.240148+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:24.005857+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:24.762654+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:25.544880+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:26.317008+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:27.077893+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:27.861434+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:28.672986+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:29.476919+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:30.254220+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:31.131393+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:31.928877+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:32.753829+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:33.570694+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:34.375167+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:35.137268+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:35.904138+00:00"
|
|
},
|
|
{
|
|
"event": "duplicate_ready",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:36.672446+00:00"
|
|
}
|
|
],
|
|
"version": 2,
|
|
"old_audit": [
|
|
{
|
|
"event": "accepted",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:17.238597+00:00"
|
|
},
|
|
{
|
|
"event": "claimed",
|
|
"details": {
|
|
"pid": 17635,
|
|
"fence": "500e09ff-0f9d-4a59-9084-ecb7c0fe2bd0",
|
|
"attempt": 1
|
|
},
|
|
"created_at": "2026-10-17 17:56:17.247016+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:20.909821+00:00"
|
|
},
|
|
{
|
|
"event": "publication_fenced",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:21.215333+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:21.714782+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:22.476928+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:23.238615+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:24.004298+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:24.821420+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:25.607833+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:26.375490+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:27.127431+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:27.911537+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:28.724080+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:29.478445+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:30.260567+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:31.132939+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:31.927497+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:32.752154+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:33.569033+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:34.373552+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:35.196087+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:36.003234+00:00"
|
|
},
|
|
{
|
|
"event": "stale_delivery",
|
|
"details": {},
|
|
"created_at": "2026-10-17 17:56:36.786981+00:00"
|
|
}
|
|
],
|
|
"duplicates": 44
|
|
}
|
|
],
|
|
"accepted": 11,
|
|
"lost_confirmed": 0,
|
|
"stale_overwrites": 0,
|
|
"worker_cgroup": {
|
|
"memory.current": "83656704",
|
|
"memory.peak": "218660864",
|
|
"memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0",
|
|
"cpu.stat": "usage_usec 35889545\nuser_usec 33648322\nsystem_usec 2241223\nnice_usec 0\nnr_periods 213\nnr_throttled 48\nthrottled_usec 221931\nnr_bursts 0\nburst_usec 0"
|
|
},
|
|
"logs_credentials_absent": true
|
|
}
|