forked from mtusi2026/mtusi2026lab3
1626 lines
34 KiB
JSON
1626 lines
34 KiB
JSON
{
|
|
"mode": "failover",
|
|
"started": "2026-10-20T07:56:31+0300",
|
|
"profile": {
|
|
"workers": 4,
|
|
"loop": "closed: POST message (retry same key until 201) + GET limit=1",
|
|
"client_timeout_seconds": 5,
|
|
"phases_seconds": {
|
|
"before": 8,
|
|
"outage": 10,
|
|
"after": 8
|
|
}
|
|
},
|
|
"runs": [
|
|
{
|
|
"scenario": "SIGTERM messenger-lab3-api-1",
|
|
"target_instance": "7535b2ed4434",
|
|
"workers": 4,
|
|
"events_seconds": {
|
|
"action_start": 10.137,
|
|
"action_end": 11.524,
|
|
"restore_start": 21.525,
|
|
"restore_end": 23.423,
|
|
"end": 31.428
|
|
},
|
|
"instances_before": [
|
|
"6d2f6cd2b04a",
|
|
"7535b2ed4434"
|
|
],
|
|
"instances_after": [
|
|
"6d2f6cd2b04a",
|
|
"eb1dace786c6"
|
|
],
|
|
"target_responses_while_stopped": 0,
|
|
"restored_replica_in_rotation_seconds": 0.971,
|
|
"instances_seen_from_action": {
|
|
"6d2f6cd2b04a": [
|
|
-10.128,
|
|
21.291
|
|
],
|
|
"7535b2ed4434": [
|
|
-10.128,
|
|
0.032
|
|
],
|
|
"eb1dace786c6": [
|
|
14.257,
|
|
21.289
|
|
]
|
|
},
|
|
"phases": {
|
|
"before": {
|
|
"seconds": 10.137,
|
|
"write": {
|
|
"ok": 3868,
|
|
"errors": 0,
|
|
"ok_rps": 381.6,
|
|
"latency_ms": {
|
|
"p50": 5.0,
|
|
"p95": 9.5,
|
|
"p99": 18.4
|
|
}
|
|
},
|
|
"read": {
|
|
"ok": 3868,
|
|
"errors": 0,
|
|
"ok_rps": 381.6,
|
|
"latency_ms": {
|
|
"p50": 3.8,
|
|
"p95": 7.2,
|
|
"p99": 22.3
|
|
}
|
|
}
|
|
},
|
|
"during": {
|
|
"seconds": 13.286,
|
|
"write": {
|
|
"ok": 3679,
|
|
"errors": 0,
|
|
"ok_rps": 276.9,
|
|
"latency_ms": {
|
|
"p50": 6.8,
|
|
"p95": 13.1,
|
|
"p99": 28.4
|
|
}
|
|
},
|
|
"read": {
|
|
"ok": 3678,
|
|
"errors": 0,
|
|
"ok_rps": 276.8,
|
|
"latency_ms": {
|
|
"p50": 5.4,
|
|
"p95": 10.9,
|
|
"p99": 25.6
|
|
}
|
|
}
|
|
},
|
|
"after": {
|
|
"seconds": 8.005,
|
|
"write": {
|
|
"ok": 3193,
|
|
"errors": 0,
|
|
"ok_rps": 398.9,
|
|
"latency_ms": {
|
|
"p50": 4.9,
|
|
"p95": 8.7,
|
|
"p99": 18.6
|
|
}
|
|
},
|
|
"read": {
|
|
"ok": 3194,
|
|
"errors": 0,
|
|
"ok_rps": 399.0,
|
|
"latency_ms": {
|
|
"p50": 3.8,
|
|
"p95": 6.5,
|
|
"p99": 14.5
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"recovery": {
|
|
"write_errors": 0,
|
|
"error_kinds": {},
|
|
"write_recovery_seconds": 0.0,
|
|
"max_write_gap_seconds": 0.063
|
|
},
|
|
"integrity": {
|
|
"messages_in_history": 10740,
|
|
"confirmed_unique": 10740,
|
|
"confirmed_missing_or_changed": 0,
|
|
"duplicate_ids": 0,
|
|
"duplicate_texts": 0,
|
|
"retried_writes_confirmed_as_replay": 0
|
|
},
|
|
"container_stats_before_action": [
|
|
"messenger-lab3-nginx-1\t9.28%\t3.09MiB / 64MiB",
|
|
"messenger-lab3-postgres-1\t27.64%\t67.91MiB / 512MiB",
|
|
"messenger-lab3-api-1\t83.48%\t48.34MiB / 512MiB",
|
|
"messenger-lab3-api-2\t85.91%\t46.54MiB / 512MiB"
|
|
],
|
|
"action_output": [
|
|
"messenger-lab3-api-1"
|
|
],
|
|
"restore_output_tail": [],
|
|
"timeline": [
|
|
{
|
|
"t": 0,
|
|
"ok": 608,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 302,
|
|
"7535b2ed4434": 306
|
|
}
|
|
},
|
|
{
|
|
"t": 1,
|
|
"ok": 790,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 397,
|
|
"7535b2ed4434": 393
|
|
}
|
|
},
|
|
{
|
|
"t": 2,
|
|
"ok": 619,
|
|
"errors": 0,
|
|
"instances": {
|
|
"7535b2ed4434": 310,
|
|
"6d2f6cd2b04a": 309
|
|
}
|
|
},
|
|
{
|
|
"t": 3,
|
|
"ok": 867,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 434,
|
|
"7535b2ed4434": 433
|
|
}
|
|
},
|
|
{
|
|
"t": 4,
|
|
"ok": 875,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 437,
|
|
"7535b2ed4434": 438
|
|
}
|
|
},
|
|
{
|
|
"t": 5,
|
|
"ok": 966,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 483,
|
|
"7535b2ed4434": 483
|
|
}
|
|
},
|
|
{
|
|
"t": 6,
|
|
"ok": 647,
|
|
"errors": 0,
|
|
"instances": {
|
|
"7535b2ed4434": 323,
|
|
"6d2f6cd2b04a": 324
|
|
}
|
|
},
|
|
{
|
|
"t": 7,
|
|
"ok": 732,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 366,
|
|
"7535b2ed4434": 366
|
|
}
|
|
},
|
|
{
|
|
"t": 8,
|
|
"ok": 687,
|
|
"errors": 0,
|
|
"instances": {
|
|
"7535b2ed4434": 344,
|
|
"6d2f6cd2b04a": 343
|
|
}
|
|
},
|
|
{
|
|
"t": 9,
|
|
"ok": 839,
|
|
"errors": 0,
|
|
"instances": {
|
|
"7535b2ed4434": 419,
|
|
"6d2f6cd2b04a": 420
|
|
}
|
|
},
|
|
{
|
|
"t": 10,
|
|
"ok": 607,
|
|
"errors": 0,
|
|
"instances": {
|
|
"7535b2ed4434": 64,
|
|
"6d2f6cd2b04a": 543
|
|
}
|
|
},
|
|
{
|
|
"t": 11,
|
|
"ok": 431,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 431
|
|
}
|
|
},
|
|
{
|
|
"t": 12,
|
|
"ok": 567,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 567
|
|
}
|
|
},
|
|
{
|
|
"t": 13,
|
|
"ok": 664,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 664
|
|
}
|
|
},
|
|
{
|
|
"t": 14,
|
|
"ok": 659,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 659
|
|
}
|
|
},
|
|
{
|
|
"t": 15,
|
|
"ok": 664,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 664
|
|
}
|
|
},
|
|
{
|
|
"t": 16,
|
|
"ok": 620,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 620
|
|
}
|
|
},
|
|
{
|
|
"t": 17,
|
|
"ok": 482,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 482
|
|
}
|
|
},
|
|
{
|
|
"t": 18,
|
|
"ok": 567,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 567
|
|
}
|
|
},
|
|
{
|
|
"t": 19,
|
|
"ok": 661,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 661
|
|
}
|
|
},
|
|
{
|
|
"t": 20,
|
|
"ok": 548,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 548
|
|
}
|
|
},
|
|
{
|
|
"t": 21,
|
|
"ok": 470,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 470
|
|
}
|
|
},
|
|
{
|
|
"t": 22,
|
|
"ok": 358,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 358
|
|
}
|
|
},
|
|
{
|
|
"t": 23,
|
|
"ok": 542,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 542
|
|
}
|
|
},
|
|
{
|
|
"t": 24,
|
|
"ok": 748,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 491,
|
|
"eb1dace786c6": 257
|
|
}
|
|
},
|
|
{
|
|
"t": 25,
|
|
"ok": 904,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 452,
|
|
"6d2f6cd2b04a": 452
|
|
}
|
|
},
|
|
{
|
|
"t": 26,
|
|
"ok": 904,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 452,
|
|
"6d2f6cd2b04a": 452
|
|
}
|
|
},
|
|
{
|
|
"t": 27,
|
|
"ok": 721,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 362,
|
|
"6d2f6cd2b04a": 359
|
|
}
|
|
},
|
|
{
|
|
"t": 28,
|
|
"ok": 525,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 263,
|
|
"eb1dace786c6": 262
|
|
}
|
|
},
|
|
{
|
|
"t": 29,
|
|
"ok": 898,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 450,
|
|
"eb1dace786c6": 448
|
|
}
|
|
},
|
|
{
|
|
"t": 30,
|
|
"ok": 915,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 457,
|
|
"eb1dace786c6": 458
|
|
}
|
|
},
|
|
{
|
|
"t": 31,
|
|
"ok": 395,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 198,
|
|
"eb1dace786c6": 197
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"scenario": "SIGTERM messenger-lab3-api-2",
|
|
"target_instance": "6d2f6cd2b04a",
|
|
"workers": 4,
|
|
"events_seconds": {
|
|
"action_start": 10.128,
|
|
"action_end": 11.576,
|
|
"restore_start": 21.577,
|
|
"restore_end": 23.313,
|
|
"end": 31.319
|
|
},
|
|
"instances_before": [
|
|
"6d2f6cd2b04a",
|
|
"eb1dace786c6"
|
|
],
|
|
"instances_after": [
|
|
"577316c523cb",
|
|
"eb1dace786c6"
|
|
],
|
|
"target_responses_while_stopped": 0,
|
|
"restored_replica_in_rotation_seconds": 3.736,
|
|
"instances_seen_from_action": {
|
|
"6d2f6cd2b04a": [
|
|
-10.122,
|
|
0.079
|
|
],
|
|
"eb1dace786c6": [
|
|
-10.121,
|
|
21.19
|
|
],
|
|
"577316c523cb": [
|
|
16.921,
|
|
21.191
|
|
]
|
|
},
|
|
"phases": {
|
|
"before": {
|
|
"seconds": 10.128,
|
|
"write": {
|
|
"ok": 4064,
|
|
"errors": 0,
|
|
"ok_rps": 401.3,
|
|
"latency_ms": {
|
|
"p50": 5.0,
|
|
"p95": 8.4,
|
|
"p99": 22.1
|
|
}
|
|
},
|
|
"read": {
|
|
"ok": 4062,
|
|
"errors": 0,
|
|
"ok_rps": 401.1,
|
|
"latency_ms": {
|
|
"p50": 3.8,
|
|
"p95": 6.2,
|
|
"p99": 12.9
|
|
}
|
|
}
|
|
},
|
|
"during": {
|
|
"seconds": 13.185,
|
|
"write": {
|
|
"ok": 3722,
|
|
"errors": 0,
|
|
"ok_rps": 282.3,
|
|
"latency_ms": {
|
|
"p50": 6.7,
|
|
"p95": 12.6,
|
|
"p99": 34.7
|
|
}
|
|
},
|
|
"read": {
|
|
"ok": 3723,
|
|
"errors": 0,
|
|
"ok_rps": 282.4,
|
|
"latency_ms": {
|
|
"p50": 5.4,
|
|
"p95": 9.9,
|
|
"p99": 17.3
|
|
}
|
|
}
|
|
},
|
|
"after": {
|
|
"seconds": 8.006,
|
|
"write": {
|
|
"ok": 2901,
|
|
"errors": 0,
|
|
"ok_rps": 362.4,
|
|
"latency_ms": {
|
|
"p50": 5.3,
|
|
"p95": 10.5,
|
|
"p99": 19.2
|
|
}
|
|
},
|
|
"read": {
|
|
"ok": 2902,
|
|
"errors": 0,
|
|
"ok_rps": 362.5,
|
|
"latency_ms": {
|
|
"p50": 4.2,
|
|
"p95": 7.9,
|
|
"p99": 14.7
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"recovery": {
|
|
"write_errors": 0,
|
|
"error_kinds": {},
|
|
"write_recovery_seconds": 0.0,
|
|
"max_write_gap_seconds": 0.071
|
|
},
|
|
"integrity": {
|
|
"messages_in_history": 10687,
|
|
"confirmed_unique": 10687,
|
|
"confirmed_missing_or_changed": 0,
|
|
"duplicate_ids": 0,
|
|
"duplicate_texts": 0,
|
|
"retried_writes_confirmed_as_replay": 0
|
|
},
|
|
"container_stats_before_action": [
|
|
"messenger-lab3-nginx-1\t9.92%\t3.406MiB / 64MiB",
|
|
"messenger-lab3-postgres-1\t29.31%\t75.47MiB / 512MiB",
|
|
"messenger-lab3-api-1\t83.72%\t47.36MiB / 512MiB",
|
|
"messenger-lab3-api-2\t84.84%\t48.22MiB / 512MiB"
|
|
],
|
|
"action_output": [
|
|
"messenger-lab3-api-2"
|
|
],
|
|
"restore_output_tail": [],
|
|
"timeline": [
|
|
{
|
|
"t": 0,
|
|
"ok": 897,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 449,
|
|
"eb1dace786c6": 448
|
|
}
|
|
},
|
|
{
|
|
"t": 1,
|
|
"ok": 743,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 372,
|
|
"6d2f6cd2b04a": 371
|
|
}
|
|
},
|
|
{
|
|
"t": 2,
|
|
"ok": 911,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 455,
|
|
"6d2f6cd2b04a": 456
|
|
}
|
|
},
|
|
{
|
|
"t": 3,
|
|
"ok": 881,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 440,
|
|
"eb1dace786c6": 441
|
|
}
|
|
},
|
|
{
|
|
"t": 4,
|
|
"ok": 837,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 419,
|
|
"eb1dace786c6": 418
|
|
}
|
|
},
|
|
{
|
|
"t": 5,
|
|
"ok": 863,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 431,
|
|
"6d2f6cd2b04a": 432
|
|
}
|
|
},
|
|
{
|
|
"t": 6,
|
|
"ok": 542,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 272,
|
|
"6d2f6cd2b04a": 270
|
|
}
|
|
},
|
|
{
|
|
"t": 7,
|
|
"ok": 765,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 383,
|
|
"eb1dace786c6": 382
|
|
}
|
|
},
|
|
{
|
|
"t": 8,
|
|
"ok": 785,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 393,
|
|
"6d2f6cd2b04a": 392
|
|
}
|
|
},
|
|
{
|
|
"t": 9,
|
|
"ok": 796,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 398,
|
|
"eb1dace786c6": 398
|
|
}
|
|
},
|
|
{
|
|
"t": 10,
|
|
"ok": 666,
|
|
"errors": 0,
|
|
"instances": {
|
|
"6d2f6cd2b04a": 86,
|
|
"eb1dace786c6": 580
|
|
}
|
|
},
|
|
{
|
|
"t": 11,
|
|
"ok": 427,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 427
|
|
}
|
|
},
|
|
{
|
|
"t": 12,
|
|
"ok": 493,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 493
|
|
}
|
|
},
|
|
{
|
|
"t": 13,
|
|
"ok": 649,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 649
|
|
}
|
|
},
|
|
{
|
|
"t": 14,
|
|
"ok": 651,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 651
|
|
}
|
|
},
|
|
{
|
|
"t": 15,
|
|
"ok": 643,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 643
|
|
}
|
|
},
|
|
{
|
|
"t": 16,
|
|
"ok": 570,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 570
|
|
}
|
|
},
|
|
{
|
|
"t": 17,
|
|
"ok": 444,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 444
|
|
}
|
|
},
|
|
{
|
|
"t": 18,
|
|
"ok": 637,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 637
|
|
}
|
|
},
|
|
{
|
|
"t": 19,
|
|
"ok": 648,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 648
|
|
}
|
|
},
|
|
{
|
|
"t": 20,
|
|
"ok": 656,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 656
|
|
}
|
|
},
|
|
{
|
|
"t": 21,
|
|
"ok": 550,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 550
|
|
}
|
|
},
|
|
{
|
|
"t": 22,
|
|
"ok": 431,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 431
|
|
}
|
|
},
|
|
{
|
|
"t": 23,
|
|
"ok": 463,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 463
|
|
}
|
|
},
|
|
{
|
|
"t": 24,
|
|
"ok": 623,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 623
|
|
}
|
|
},
|
|
{
|
|
"t": 25,
|
|
"ok": 648,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 648
|
|
}
|
|
},
|
|
{
|
|
"t": 26,
|
|
"ok": 566,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 566
|
|
}
|
|
},
|
|
{
|
|
"t": 27,
|
|
"ok": 823,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 427,
|
|
"577316c523cb": 396
|
|
}
|
|
},
|
|
{
|
|
"t": 28,
|
|
"ok": 548,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 274,
|
|
"577316c523cb": 274
|
|
}
|
|
},
|
|
{
|
|
"t": 29,
|
|
"ok": 941,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 470,
|
|
"577316c523cb": 471
|
|
}
|
|
},
|
|
{
|
|
"t": 30,
|
|
"ok": 977,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 490,
|
|
"577316c523cb": 487
|
|
}
|
|
},
|
|
{
|
|
"t": 31,
|
|
"ok": 300,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 151,
|
|
"eb1dace786c6": 149
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"scenario": "SIGKILL messenger-lab3-api-1",
|
|
"target_instance": "eb1dace786c6",
|
|
"workers": 4,
|
|
"events_seconds": {
|
|
"action_start": 9.738,
|
|
"action_end": 9.957,
|
|
"restore_start": 19.957,
|
|
"restore_end": 21.841,
|
|
"end": 29.849
|
|
},
|
|
"instances_before": [
|
|
"577316c523cb",
|
|
"eb1dace786c6"
|
|
],
|
|
"instances_after": [
|
|
"506b547ba744",
|
|
"577316c523cb"
|
|
],
|
|
"target_responses_while_stopped": 0,
|
|
"restored_replica_in_rotation_seconds": 0.79,
|
|
"instances_seen_from_action": {
|
|
"eb1dace786c6": [
|
|
-9.73,
|
|
0.024
|
|
],
|
|
"577316c523cb": [
|
|
-9.73,
|
|
20.11
|
|
],
|
|
"506b547ba744": [
|
|
12.893,
|
|
20.111
|
|
]
|
|
},
|
|
"phases": {
|
|
"before": {
|
|
"seconds": 9.738,
|
|
"write": {
|
|
"ok": 4162,
|
|
"errors": 0,
|
|
"ok_rps": 427.4,
|
|
"latency_ms": {
|
|
"p50": 4.6,
|
|
"p95": 7.6,
|
|
"p99": 14.6
|
|
}
|
|
},
|
|
"read": {
|
|
"ok": 4161,
|
|
"errors": 0,
|
|
"ok_rps": 427.3,
|
|
"latency_ms": {
|
|
"p50": 3.6,
|
|
"p95": 5.8,
|
|
"p99": 11.7
|
|
}
|
|
}
|
|
},
|
|
"during": {
|
|
"seconds": 12.103,
|
|
"write": {
|
|
"ok": 3406,
|
|
"errors": 1,
|
|
"ok_rps": 281.4,
|
|
"latency_ms": {
|
|
"p50": 6.9,
|
|
"p95": 12.7,
|
|
"p99": 22.9
|
|
}
|
|
},
|
|
"read": {
|
|
"ok": 3404,
|
|
"errors": 0,
|
|
"ok_rps": 281.3,
|
|
"latency_ms": {
|
|
"p50": 5.6,
|
|
"p95": 10.1,
|
|
"p99": 16.4
|
|
}
|
|
}
|
|
},
|
|
"after": {
|
|
"seconds": 8.008,
|
|
"write": {
|
|
"ok": 3269,
|
|
"errors": 0,
|
|
"ok_rps": 408.2,
|
|
"latency_ms": {
|
|
"p50": 4.7,
|
|
"p95": 8.1,
|
|
"p99": 21.3
|
|
}
|
|
},
|
|
"read": {
|
|
"ok": 3271,
|
|
"errors": 0,
|
|
"ok_rps": 408.5,
|
|
"latency_ms": {
|
|
"p50": 3.6,
|
|
"p95": 6.1,
|
|
"p99": 15.5
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"recovery": {
|
|
"write_errors": 1,
|
|
"error_kinds": {
|
|
"HTTP 503": 1
|
|
},
|
|
"write_recovery_seconds": 0.0,
|
|
"max_write_gap_seconds": 0.07
|
|
},
|
|
"integrity": {
|
|
"messages_in_history": 10837,
|
|
"confirmed_unique": 10837,
|
|
"confirmed_missing_or_changed": 0,
|
|
"duplicate_ids": 0,
|
|
"duplicate_texts": 0,
|
|
"retried_writes_confirmed_as_replay": 0
|
|
},
|
|
"container_stats_before_action": [
|
|
"messenger-lab3-nginx-1\t9.89%\t2.816MiB / 64MiB",
|
|
"messenger-lab3-postgres-1\t27.37%\t80.08MiB / 512MiB",
|
|
"messenger-lab3-api-1\t84.41%\t48.93MiB / 512MiB",
|
|
"messenger-lab3-api-2\t84.27%\t47.33MiB / 512MiB"
|
|
],
|
|
"action_output": [
|
|
"messenger-lab3-api-1"
|
|
],
|
|
"restore_output_tail": [],
|
|
"timeline": [
|
|
{
|
|
"t": 0,
|
|
"ok": 826,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 413,
|
|
"577316c523cb": 413
|
|
}
|
|
},
|
|
{
|
|
"t": 1,
|
|
"ok": 645,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 322,
|
|
"eb1dace786c6": 323
|
|
}
|
|
},
|
|
{
|
|
"t": 2,
|
|
"ok": 963,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 481,
|
|
"577316c523cb": 482
|
|
}
|
|
},
|
|
{
|
|
"t": 3,
|
|
"ok": 828,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 414,
|
|
"577316c523cb": 414
|
|
}
|
|
},
|
|
{
|
|
"t": 4,
|
|
"ok": 899,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 450,
|
|
"577316c523cb": 449
|
|
}
|
|
},
|
|
{
|
|
"t": 5,
|
|
"ok": 852,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 426,
|
|
"eb1dace786c6": 426
|
|
}
|
|
},
|
|
{
|
|
"t": 6,
|
|
"ok": 704,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 352,
|
|
"577316c523cb": 352
|
|
}
|
|
},
|
|
{
|
|
"t": 7,
|
|
"ok": 918,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 459,
|
|
"577316c523cb": 459
|
|
}
|
|
},
|
|
{
|
|
"t": 8,
|
|
"ok": 975,
|
|
"errors": 0,
|
|
"instances": {
|
|
"eb1dace786c6": 487,
|
|
"577316c523cb": 488
|
|
}
|
|
},
|
|
{
|
|
"t": 9,
|
|
"ok": 853,
|
|
"errors": 1,
|
|
"instances": {
|
|
"eb1dace786c6": 365,
|
|
"577316c523cb": 488
|
|
}
|
|
},
|
|
{
|
|
"t": 10,
|
|
"ok": 621,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 621
|
|
}
|
|
},
|
|
{
|
|
"t": 11,
|
|
"ok": 618,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 618
|
|
}
|
|
},
|
|
{
|
|
"t": 12,
|
|
"ok": 474,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 474
|
|
}
|
|
},
|
|
{
|
|
"t": 13,
|
|
"ok": 569,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 569
|
|
}
|
|
},
|
|
{
|
|
"t": 14,
|
|
"ok": 633,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 633
|
|
}
|
|
},
|
|
{
|
|
"t": 15,
|
|
"ok": 548,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 548
|
|
}
|
|
},
|
|
{
|
|
"t": 16,
|
|
"ok": 592,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 592
|
|
}
|
|
},
|
|
{
|
|
"t": 17,
|
|
"ok": 459,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 459
|
|
}
|
|
},
|
|
{
|
|
"t": 18,
|
|
"ok": 570,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 570
|
|
}
|
|
},
|
|
{
|
|
"t": 19,
|
|
"ok": 612,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 612
|
|
}
|
|
},
|
|
{
|
|
"t": 20,
|
|
"ok": 487,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 487
|
|
}
|
|
},
|
|
{
|
|
"t": 21,
|
|
"ok": 580,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 580
|
|
}
|
|
},
|
|
{
|
|
"t": 22,
|
|
"ok": 742,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 568,
|
|
"506b547ba744": 174
|
|
}
|
|
},
|
|
{
|
|
"t": 23,
|
|
"ok": 605,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 303,
|
|
"506b547ba744": 302
|
|
}
|
|
},
|
|
{
|
|
"t": 24,
|
|
"ok": 959,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 480,
|
|
"506b547ba744": 479
|
|
}
|
|
},
|
|
{
|
|
"t": 25,
|
|
"ok": 921,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 461,
|
|
"577316c523cb": 460
|
|
}
|
|
},
|
|
{
|
|
"t": 26,
|
|
"ok": 745,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 373,
|
|
"506b547ba744": 372
|
|
}
|
|
},
|
|
{
|
|
"t": 27,
|
|
"ok": 889,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 446,
|
|
"577316c523cb": 443
|
|
}
|
|
},
|
|
{
|
|
"t": 28,
|
|
"ok": 748,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 375,
|
|
"506b547ba744": 373
|
|
}
|
|
},
|
|
{
|
|
"t": 29,
|
|
"ok": 839,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 419,
|
|
"506b547ba744": 420
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"scenario": "SIGKILL messenger-lab3-api-2",
|
|
"target_instance": "577316c523cb",
|
|
"workers": 4,
|
|
"events_seconds": {
|
|
"action_start": 9.946,
|
|
"action_end": 10.207,
|
|
"restore_start": 20.209,
|
|
"restore_end": 22.126,
|
|
"end": 30.136
|
|
},
|
|
"instances_before": [
|
|
"506b547ba744",
|
|
"577316c523cb"
|
|
],
|
|
"instances_after": [
|
|
"1c20e7f6dbde",
|
|
"506b547ba744"
|
|
],
|
|
"target_responses_while_stopped": 0,
|
|
"restored_replica_in_rotation_seconds": 0.0,
|
|
"instances_seen_from_action": {
|
|
"506b547ba744": [
|
|
-9.94,
|
|
20.189
|
|
],
|
|
"577316c523cb": [
|
|
-9.939,
|
|
0.028
|
|
],
|
|
"1c20e7f6dbde": [
|
|
11.911,
|
|
20.19
|
|
]
|
|
},
|
|
"phases": {
|
|
"before": {
|
|
"seconds": 9.946,
|
|
"write": {
|
|
"ok": 3853,
|
|
"errors": 0,
|
|
"ok_rps": 387.4,
|
|
"latency_ms": {
|
|
"p50": 4.8,
|
|
"p95": 9.1,
|
|
"p99": 34.8
|
|
}
|
|
},
|
|
"read": {
|
|
"ok": 3850,
|
|
"errors": 0,
|
|
"ok_rps": 387.1,
|
|
"latency_ms": {
|
|
"p50": 3.7,
|
|
"p95": 7.0,
|
|
"p99": 19.5
|
|
}
|
|
}
|
|
},
|
|
"during": {
|
|
"seconds": 12.18,
|
|
"write": {
|
|
"ok": 3370,
|
|
"errors": 3,
|
|
"ok_rps": 276.7,
|
|
"latency_ms": {
|
|
"p50": 6.8,
|
|
"p95": 12.8,
|
|
"p99": 26.8
|
|
}
|
|
},
|
|
"read": {
|
|
"ok": 3372,
|
|
"errors": 0,
|
|
"ok_rps": 276.8,
|
|
"latency_ms": {
|
|
"p50": 5.5,
|
|
"p95": 11.0,
|
|
"p99": 28.6
|
|
}
|
|
}
|
|
},
|
|
"after": {
|
|
"seconds": 8.01,
|
|
"write": {
|
|
"ok": 2989,
|
|
"errors": 0,
|
|
"ok_rps": 373.2,
|
|
"latency_ms": {
|
|
"p50": 5.0,
|
|
"p95": 10.1,
|
|
"p99": 35.1
|
|
}
|
|
},
|
|
"read": {
|
|
"ok": 2990,
|
|
"errors": 0,
|
|
"ok_rps": 373.3,
|
|
"latency_ms": {
|
|
"p50": 3.8,
|
|
"p95": 7.6,
|
|
"p99": 22.2
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"recovery": {
|
|
"write_errors": 3,
|
|
"error_kinds": {
|
|
"HTTP 503": 3
|
|
},
|
|
"write_recovery_seconds": 0.006,
|
|
"max_write_gap_seconds": 0.061
|
|
},
|
|
"integrity": {
|
|
"messages_in_history": 10212,
|
|
"confirmed_unique": 10212,
|
|
"confirmed_missing_or_changed": 0,
|
|
"duplicate_ids": 0,
|
|
"duplicate_texts": 0,
|
|
"retried_writes_confirmed_as_replay": 0
|
|
},
|
|
"container_stats_before_action": [
|
|
"messenger-lab3-nginx-1\t9.12%\t2.887MiB / 64MiB",
|
|
"messenger-lab3-postgres-1\t27.19%\t90.89MiB / 512MiB",
|
|
"messenger-lab3-api-1\t80.43%\t47.9MiB / 512MiB",
|
|
"messenger-lab3-api-2\t80.26%\t50.77MiB / 512MiB"
|
|
],
|
|
"action_output": [
|
|
"messenger-lab3-api-2"
|
|
],
|
|
"restore_output_tail": [],
|
|
"timeline": [
|
|
{
|
|
"t": 0,
|
|
"ok": 925,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 462,
|
|
"577316c523cb": 463
|
|
}
|
|
},
|
|
{
|
|
"t": 1,
|
|
"ok": 665,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 333,
|
|
"577316c523cb": 332
|
|
}
|
|
},
|
|
{
|
|
"t": 2,
|
|
"ok": 856,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 428,
|
|
"506b547ba744": 428
|
|
}
|
|
},
|
|
{
|
|
"t": 3,
|
|
"ok": 641,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 321,
|
|
"506b547ba744": 320
|
|
}
|
|
},
|
|
{
|
|
"t": 4,
|
|
"ok": 868,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 434,
|
|
"506b547ba744": 434
|
|
}
|
|
},
|
|
{
|
|
"t": 5,
|
|
"ok": 874,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 437,
|
|
"577316c523cb": 437
|
|
}
|
|
},
|
|
{
|
|
"t": 6,
|
|
"ok": 753,
|
|
"errors": 0,
|
|
"instances": {
|
|
"577316c523cb": 378,
|
|
"506b547ba744": 375
|
|
}
|
|
},
|
|
{
|
|
"t": 7,
|
|
"ok": 875,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 439,
|
|
"577316c523cb": 436
|
|
}
|
|
},
|
|
{
|
|
"t": 8,
|
|
"ok": 649,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 325,
|
|
"577316c523cb": 324
|
|
}
|
|
},
|
|
{
|
|
"t": 9,
|
|
"ok": 622,
|
|
"errors": 3,
|
|
"instances": {
|
|
"577316c523cb": 309,
|
|
"506b547ba744": 313
|
|
}
|
|
},
|
|
{
|
|
"t": 10,
|
|
"ok": 532,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 532
|
|
}
|
|
},
|
|
{
|
|
"t": 11,
|
|
"ok": 653,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 653
|
|
}
|
|
},
|
|
{
|
|
"t": 12,
|
|
"ok": 361,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 361
|
|
}
|
|
},
|
|
{
|
|
"t": 13,
|
|
"ok": 626,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 626
|
|
}
|
|
},
|
|
{
|
|
"t": 14,
|
|
"ok": 618,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 618
|
|
}
|
|
},
|
|
{
|
|
"t": 15,
|
|
"ok": 649,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 649
|
|
}
|
|
},
|
|
{
|
|
"t": 16,
|
|
"ok": 651,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 651
|
|
}
|
|
},
|
|
{
|
|
"t": 17,
|
|
"ok": 461,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 461
|
|
}
|
|
},
|
|
{
|
|
"t": 18,
|
|
"ok": 516,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 516
|
|
}
|
|
},
|
|
{
|
|
"t": 19,
|
|
"ok": 611,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 611
|
|
}
|
|
},
|
|
{
|
|
"t": 20,
|
|
"ok": 436,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 436
|
|
}
|
|
},
|
|
{
|
|
"t": 21,
|
|
"ok": 487,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 451,
|
|
"1c20e7f6dbde": 36
|
|
}
|
|
},
|
|
{
|
|
"t": 22,
|
|
"ok": 769,
|
|
"errors": 0,
|
|
"instances": {
|
|
"1c20e7f6dbde": 385,
|
|
"506b547ba744": 384
|
|
}
|
|
},
|
|
{
|
|
"t": 23,
|
|
"ok": 614,
|
|
"errors": 0,
|
|
"instances": {
|
|
"1c20e7f6dbde": 307,
|
|
"506b547ba744": 307
|
|
}
|
|
},
|
|
{
|
|
"t": 24,
|
|
"ok": 804,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 402,
|
|
"1c20e7f6dbde": 402
|
|
}
|
|
},
|
|
{
|
|
"t": 25,
|
|
"ok": 717,
|
|
"errors": 0,
|
|
"instances": {
|
|
"1c20e7f6dbde": 358,
|
|
"506b547ba744": 359
|
|
}
|
|
},
|
|
{
|
|
"t": 26,
|
|
"ok": 827,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 415,
|
|
"1c20e7f6dbde": 412
|
|
}
|
|
},
|
|
{
|
|
"t": 27,
|
|
"ok": 735,
|
|
"errors": 0,
|
|
"instances": {
|
|
"1c20e7f6dbde": 369,
|
|
"506b547ba744": 366
|
|
}
|
|
},
|
|
{
|
|
"t": 28,
|
|
"ok": 648,
|
|
"errors": 0,
|
|
"instances": {
|
|
"1c20e7f6dbde": 326,
|
|
"506b547ba744": 322
|
|
}
|
|
},
|
|
{
|
|
"t": 29,
|
|
"ok": 851,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 427,
|
|
"1c20e7f6dbde": 424
|
|
}
|
|
},
|
|
{
|
|
"t": 30,
|
|
"ok": 130,
|
|
"errors": 0,
|
|
"instances": {
|
|
"506b547ba744": 65,
|
|
"1c20e7f6dbde": 65
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|