Files
mtusi2026lab6-fork/docs/lab5/evidence/failover_series_kill.json
T

299 lines
6.4 KiB
JSON

{
"scenario": "stop each API replica in turn under load, restore before the next",
"load": {
"writers": 4,
"readers": 2,
"client_timeout_s": 2.0,
"write_retry": "same Idempotency-Key and body until 201"
},
"replicas": [
"3a0e06f14a63",
"4b1faad0399c"
],
"rounds": [
{
"target_instance": "3a0e06f14a63",
"mode": "kill",
"lb_up_before": [
"api1",
"api2"
],
"stop_hook_seconds": 1.2,
"lb_detect_down_seconds": 2.39,
"target_last_served_after_stop_seconds": 0.56,
"target_served_after_hook_returned": false,
"errors_during_outage": 3,
"error_kinds": [
"502"
],
"error_window_seconds": 0.64,
"restore_hook_seconds": 7.12,
"lb_back_up_seconds": 1.42,
"target_back_in_rotation_seconds": 7.06,
"windows": {
"before": {
"from_s": 0,
"to_s": 5.0,
"requests": 1402,
"ok": 1402,
"errors": 0,
"error_ratio": 0.0,
"ok_rps": 280.4,
"write_ok_rps": 161.4,
"latency_ms_ok": {
"p50": 16.0,
"p95": 78.0,
"p99": 94.0
},
"instances": [
"3a0e06f14a63",
"4b1faad0399c"
]
},
"outage_first_10s": {
"from_s": 5.0,
"to_s": 15.0,
"requests": 901,
"ok": 898,
"errors": 3,
"error_ratio": 0.0033,
"ok_rps": 89.8,
"write_ok_rps": 52.6,
"latency_ms_ok": {
"p50": 32.0,
"p95": 110.0,
"p99": 1015.0
},
"instances": [
"3a0e06f14a63",
"4b1faad0399c"
]
},
"outage_rest": {
"from_s": 15.0,
"to_s": 20.0,
"requests": 518,
"ok": 518,
"errors": 0,
"error_ratio": 0.0,
"ok_rps": 103.6,
"write_ok_rps": 60.8,
"latency_ms_ok": {
"p50": 78.0,
"p95": 110.0,
"p99": 187.0
},
"instances": [
"4b1faad0399c"
]
},
"after_restore": {
"from_s": 20.0,
"to_s": 30.44,
"requests": 1786,
"ok": 1786,
"errors": 0,
"error_ratio": 0.0,
"ok_rps": 171.1,
"write_ok_rps": 101.4,
"latency_ms_ok": {
"p50": 16.0,
"p95": 94.0,
"p99": 187.0
},
"instances": [
"3a0e06f14a63",
"4b1faad0399c"
]
}
},
"lb_timeline": [
[
0.0,
[
"api1",
"api2"
]
],
[
7.39,
[
"api1"
]
],
[
21.42,
[
"api1",
"api2"
]
],
[
21.53,
[
"api1"
]
],
[
24.02,
[
"api1",
"api2"
]
]
],
"data": {
"confirmed_writes": 2698,
"stored_messages": 2698,
"missing_or_altered": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"same_key_retries": 1,
"ok": true
}
},
{
"target_instance": "4b1faad0399c",
"mode": "kill",
"lb_up_before": [
"api1",
"api2"
],
"stop_hook_seconds": 0.83,
"lb_detect_down_seconds": 2.34,
"target_last_served_after_stop_seconds": 0.09,
"target_served_after_hook_returned": false,
"errors_during_outage": 6,
"error_kinds": [
"502"
],
"error_window_seconds": 0.28,
"restore_hook_seconds": 7.3,
"lb_back_up_seconds": 1.33,
"target_back_in_rotation_seconds": 7.28,
"windows": {
"before": {
"from_s": 0,
"to_s": 5.0,
"requests": 1302,
"ok": 1302,
"errors": 0,
"error_ratio": 0.0,
"ok_rps": 260.4,
"write_ok_rps": 153.2,
"latency_ms_ok": {
"p50": 16.0,
"p95": 78.0,
"p99": 125.0
},
"instances": [
"3a0e06f14a63",
"4b1faad0399c"
]
},
"outage_first_10s": {
"from_s": 5.0,
"to_s": 15.0,
"requests": 900,
"ok": 894,
"errors": 6,
"error_ratio": 0.0067,
"ok_rps": 89.4,
"write_ok_rps": 52.5,
"latency_ms_ok": {
"p50": 32.0,
"p95": 110.0,
"p99": 1016.0
},
"instances": [
"3a0e06f14a63",
"4b1faad0399c"
]
},
"outage_rest": {
"from_s": 15.0,
"to_s": 20.0,
"requests": 565,
"ok": 565,
"errors": 0,
"error_ratio": 0.0,
"ok_rps": 113.0,
"write_ok_rps": 66.6,
"latency_ms_ok": {
"p50": 62.0,
"p95": 109.0,
"p99": 187.0
},
"instances": [
"3a0e06f14a63"
]
},
"after_restore": {
"from_s": 20.0,
"to_s": 30.5,
"requests": 1638,
"ok": 1638,
"errors": 0,
"error_ratio": 0.0,
"ok_rps": 156.0,
"write_ok_rps": 92.5,
"latency_ms_ok": {
"p50": 16.0,
"p95": 109.0,
"p99": 188.0
},
"instances": [
"3a0e06f14a63",
"4b1faad0399c"
]
}
},
"lb_timeline": [
[
0.0,
[
"api1",
"api2"
]
],
[
7.34,
[
"api2"
]
],
[
21.33,
[
"api1",
"api2"
]
],
[
22.2,
[
"api2"
]
],
[
24.74,
[
"api1",
"api2"
]
]
],
"data": {
"confirmed_writes": 2599,
"stored_messages": 2599,
"missing_or_altered": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"same_key_retries": 4,
"ok": true
}
}
],
"pass": true
}