Files
mtusi2026lab3/evidence/experiment-failover.json
T

1626 lines
34 KiB
JSON

{
"mode": "failover",
"started": "2026-10-20T07:56:31+0300",
"profile": {
"workers": 4,
"loop": "closed: POST message (retry same key until 201) + GET limit=1",
"client_timeout_seconds": 5,
"phases_seconds": {
"before": 8,
"outage": 10,
"after": 8
}
},
"runs": [
{
"scenario": "SIGTERM messenger-lab3-api-1",
"target_instance": "7535b2ed4434",
"workers": 4,
"events_seconds": {
"action_start": 10.137,
"action_end": 11.524,
"restore_start": 21.525,
"restore_end": 23.423,
"end": 31.428
},
"instances_before": [
"6d2f6cd2b04a",
"7535b2ed4434"
],
"instances_after": [
"6d2f6cd2b04a",
"eb1dace786c6"
],
"target_responses_while_stopped": 0,
"restored_replica_in_rotation_seconds": 0.971,
"instances_seen_from_action": {
"6d2f6cd2b04a": [
-10.128,
21.291
],
"7535b2ed4434": [
-10.128,
0.032
],
"eb1dace786c6": [
14.257,
21.289
]
},
"phases": {
"before": {
"seconds": 10.137,
"write": {
"ok": 3868,
"errors": 0,
"ok_rps": 381.6,
"latency_ms": {
"p50": 5.0,
"p95": 9.5,
"p99": 18.4
}
},
"read": {
"ok": 3868,
"errors": 0,
"ok_rps": 381.6,
"latency_ms": {
"p50": 3.8,
"p95": 7.2,
"p99": 22.3
}
}
},
"during": {
"seconds": 13.286,
"write": {
"ok": 3679,
"errors": 0,
"ok_rps": 276.9,
"latency_ms": {
"p50": 6.8,
"p95": 13.1,
"p99": 28.4
}
},
"read": {
"ok": 3678,
"errors": 0,
"ok_rps": 276.8,
"latency_ms": {
"p50": 5.4,
"p95": 10.9,
"p99": 25.6
}
}
},
"after": {
"seconds": 8.005,
"write": {
"ok": 3193,
"errors": 0,
"ok_rps": 398.9,
"latency_ms": {
"p50": 4.9,
"p95": 8.7,
"p99": 18.6
}
},
"read": {
"ok": 3194,
"errors": 0,
"ok_rps": 399.0,
"latency_ms": {
"p50": 3.8,
"p95": 6.5,
"p99": 14.5
}
}
}
},
"recovery": {
"write_errors": 0,
"error_kinds": {},
"write_recovery_seconds": 0.0,
"max_write_gap_seconds": 0.063
},
"integrity": {
"messages_in_history": 10740,
"confirmed_unique": 10740,
"confirmed_missing_or_changed": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"retried_writes_confirmed_as_replay": 0
},
"container_stats_before_action": [
"messenger-lab3-nginx-1\t9.28%\t3.09MiB / 64MiB",
"messenger-lab3-postgres-1\t27.64%\t67.91MiB / 512MiB",
"messenger-lab3-api-1\t83.48%\t48.34MiB / 512MiB",
"messenger-lab3-api-2\t85.91%\t46.54MiB / 512MiB"
],
"action_output": [
"messenger-lab3-api-1"
],
"restore_output_tail": [],
"timeline": [
{
"t": 0,
"ok": 608,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 302,
"7535b2ed4434": 306
}
},
{
"t": 1,
"ok": 790,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 397,
"7535b2ed4434": 393
}
},
{
"t": 2,
"ok": 619,
"errors": 0,
"instances": {
"7535b2ed4434": 310,
"6d2f6cd2b04a": 309
}
},
{
"t": 3,
"ok": 867,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 434,
"7535b2ed4434": 433
}
},
{
"t": 4,
"ok": 875,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 437,
"7535b2ed4434": 438
}
},
{
"t": 5,
"ok": 966,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 483,
"7535b2ed4434": 483
}
},
{
"t": 6,
"ok": 647,
"errors": 0,
"instances": {
"7535b2ed4434": 323,
"6d2f6cd2b04a": 324
}
},
{
"t": 7,
"ok": 732,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 366,
"7535b2ed4434": 366
}
},
{
"t": 8,
"ok": 687,
"errors": 0,
"instances": {
"7535b2ed4434": 344,
"6d2f6cd2b04a": 343
}
},
{
"t": 9,
"ok": 839,
"errors": 0,
"instances": {
"7535b2ed4434": 419,
"6d2f6cd2b04a": 420
}
},
{
"t": 10,
"ok": 607,
"errors": 0,
"instances": {
"7535b2ed4434": 64,
"6d2f6cd2b04a": 543
}
},
{
"t": 11,
"ok": 431,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 431
}
},
{
"t": 12,
"ok": 567,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 567
}
},
{
"t": 13,
"ok": 664,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 664
}
},
{
"t": 14,
"ok": 659,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 659
}
},
{
"t": 15,
"ok": 664,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 664
}
},
{
"t": 16,
"ok": 620,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 620
}
},
{
"t": 17,
"ok": 482,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 482
}
},
{
"t": 18,
"ok": 567,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 567
}
},
{
"t": 19,
"ok": 661,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 661
}
},
{
"t": 20,
"ok": 548,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 548
}
},
{
"t": 21,
"ok": 470,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 470
}
},
{
"t": 22,
"ok": 358,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 358
}
},
{
"t": 23,
"ok": 542,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 542
}
},
{
"t": 24,
"ok": 748,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 491,
"eb1dace786c6": 257
}
},
{
"t": 25,
"ok": 904,
"errors": 0,
"instances": {
"eb1dace786c6": 452,
"6d2f6cd2b04a": 452
}
},
{
"t": 26,
"ok": 904,
"errors": 0,
"instances": {
"eb1dace786c6": 452,
"6d2f6cd2b04a": 452
}
},
{
"t": 27,
"ok": 721,
"errors": 0,
"instances": {
"eb1dace786c6": 362,
"6d2f6cd2b04a": 359
}
},
{
"t": 28,
"ok": 525,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 263,
"eb1dace786c6": 262
}
},
{
"t": 29,
"ok": 898,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 450,
"eb1dace786c6": 448
}
},
{
"t": 30,
"ok": 915,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 457,
"eb1dace786c6": 458
}
},
{
"t": 31,
"ok": 395,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 198,
"eb1dace786c6": 197
}
}
]
},
{
"scenario": "SIGTERM messenger-lab3-api-2",
"target_instance": "6d2f6cd2b04a",
"workers": 4,
"events_seconds": {
"action_start": 10.128,
"action_end": 11.576,
"restore_start": 21.577,
"restore_end": 23.313,
"end": 31.319
},
"instances_before": [
"6d2f6cd2b04a",
"eb1dace786c6"
],
"instances_after": [
"577316c523cb",
"eb1dace786c6"
],
"target_responses_while_stopped": 0,
"restored_replica_in_rotation_seconds": 3.736,
"instances_seen_from_action": {
"6d2f6cd2b04a": [
-10.122,
0.079
],
"eb1dace786c6": [
-10.121,
21.19
],
"577316c523cb": [
16.921,
21.191
]
},
"phases": {
"before": {
"seconds": 10.128,
"write": {
"ok": 4064,
"errors": 0,
"ok_rps": 401.3,
"latency_ms": {
"p50": 5.0,
"p95": 8.4,
"p99": 22.1
}
},
"read": {
"ok": 4062,
"errors": 0,
"ok_rps": 401.1,
"latency_ms": {
"p50": 3.8,
"p95": 6.2,
"p99": 12.9
}
}
},
"during": {
"seconds": 13.185,
"write": {
"ok": 3722,
"errors": 0,
"ok_rps": 282.3,
"latency_ms": {
"p50": 6.7,
"p95": 12.6,
"p99": 34.7
}
},
"read": {
"ok": 3723,
"errors": 0,
"ok_rps": 282.4,
"latency_ms": {
"p50": 5.4,
"p95": 9.9,
"p99": 17.3
}
}
},
"after": {
"seconds": 8.006,
"write": {
"ok": 2901,
"errors": 0,
"ok_rps": 362.4,
"latency_ms": {
"p50": 5.3,
"p95": 10.5,
"p99": 19.2
}
},
"read": {
"ok": 2902,
"errors": 0,
"ok_rps": 362.5,
"latency_ms": {
"p50": 4.2,
"p95": 7.9,
"p99": 14.7
}
}
}
},
"recovery": {
"write_errors": 0,
"error_kinds": {},
"write_recovery_seconds": 0.0,
"max_write_gap_seconds": 0.071
},
"integrity": {
"messages_in_history": 10687,
"confirmed_unique": 10687,
"confirmed_missing_or_changed": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"retried_writes_confirmed_as_replay": 0
},
"container_stats_before_action": [
"messenger-lab3-nginx-1\t9.92%\t3.406MiB / 64MiB",
"messenger-lab3-postgres-1\t29.31%\t75.47MiB / 512MiB",
"messenger-lab3-api-1\t83.72%\t47.36MiB / 512MiB",
"messenger-lab3-api-2\t84.84%\t48.22MiB / 512MiB"
],
"action_output": [
"messenger-lab3-api-2"
],
"restore_output_tail": [],
"timeline": [
{
"t": 0,
"ok": 897,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 449,
"eb1dace786c6": 448
}
},
{
"t": 1,
"ok": 743,
"errors": 0,
"instances": {
"eb1dace786c6": 372,
"6d2f6cd2b04a": 371
}
},
{
"t": 2,
"ok": 911,
"errors": 0,
"instances": {
"eb1dace786c6": 455,
"6d2f6cd2b04a": 456
}
},
{
"t": 3,
"ok": 881,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 440,
"eb1dace786c6": 441
}
},
{
"t": 4,
"ok": 837,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 419,
"eb1dace786c6": 418
}
},
{
"t": 5,
"ok": 863,
"errors": 0,
"instances": {
"eb1dace786c6": 431,
"6d2f6cd2b04a": 432
}
},
{
"t": 6,
"ok": 542,
"errors": 0,
"instances": {
"eb1dace786c6": 272,
"6d2f6cd2b04a": 270
}
},
{
"t": 7,
"ok": 765,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 383,
"eb1dace786c6": 382
}
},
{
"t": 8,
"ok": 785,
"errors": 0,
"instances": {
"eb1dace786c6": 393,
"6d2f6cd2b04a": 392
}
},
{
"t": 9,
"ok": 796,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 398,
"eb1dace786c6": 398
}
},
{
"t": 10,
"ok": 666,
"errors": 0,
"instances": {
"6d2f6cd2b04a": 86,
"eb1dace786c6": 580
}
},
{
"t": 11,
"ok": 427,
"errors": 0,
"instances": {
"eb1dace786c6": 427
}
},
{
"t": 12,
"ok": 493,
"errors": 0,
"instances": {
"eb1dace786c6": 493
}
},
{
"t": 13,
"ok": 649,
"errors": 0,
"instances": {
"eb1dace786c6": 649
}
},
{
"t": 14,
"ok": 651,
"errors": 0,
"instances": {
"eb1dace786c6": 651
}
},
{
"t": 15,
"ok": 643,
"errors": 0,
"instances": {
"eb1dace786c6": 643
}
},
{
"t": 16,
"ok": 570,
"errors": 0,
"instances": {
"eb1dace786c6": 570
}
},
{
"t": 17,
"ok": 444,
"errors": 0,
"instances": {
"eb1dace786c6": 444
}
},
{
"t": 18,
"ok": 637,
"errors": 0,
"instances": {
"eb1dace786c6": 637
}
},
{
"t": 19,
"ok": 648,
"errors": 0,
"instances": {
"eb1dace786c6": 648
}
},
{
"t": 20,
"ok": 656,
"errors": 0,
"instances": {
"eb1dace786c6": 656
}
},
{
"t": 21,
"ok": 550,
"errors": 0,
"instances": {
"eb1dace786c6": 550
}
},
{
"t": 22,
"ok": 431,
"errors": 0,
"instances": {
"eb1dace786c6": 431
}
},
{
"t": 23,
"ok": 463,
"errors": 0,
"instances": {
"eb1dace786c6": 463
}
},
{
"t": 24,
"ok": 623,
"errors": 0,
"instances": {
"eb1dace786c6": 623
}
},
{
"t": 25,
"ok": 648,
"errors": 0,
"instances": {
"eb1dace786c6": 648
}
},
{
"t": 26,
"ok": 566,
"errors": 0,
"instances": {
"eb1dace786c6": 566
}
},
{
"t": 27,
"ok": 823,
"errors": 0,
"instances": {
"eb1dace786c6": 427,
"577316c523cb": 396
}
},
{
"t": 28,
"ok": 548,
"errors": 0,
"instances": {
"eb1dace786c6": 274,
"577316c523cb": 274
}
},
{
"t": 29,
"ok": 941,
"errors": 0,
"instances": {
"eb1dace786c6": 470,
"577316c523cb": 471
}
},
{
"t": 30,
"ok": 977,
"errors": 0,
"instances": {
"eb1dace786c6": 490,
"577316c523cb": 487
}
},
{
"t": 31,
"ok": 300,
"errors": 0,
"instances": {
"577316c523cb": 151,
"eb1dace786c6": 149
}
}
]
},
{
"scenario": "SIGKILL messenger-lab3-api-1",
"target_instance": "eb1dace786c6",
"workers": 4,
"events_seconds": {
"action_start": 9.738,
"action_end": 9.957,
"restore_start": 19.957,
"restore_end": 21.841,
"end": 29.849
},
"instances_before": [
"577316c523cb",
"eb1dace786c6"
],
"instances_after": [
"506b547ba744",
"577316c523cb"
],
"target_responses_while_stopped": 0,
"restored_replica_in_rotation_seconds": 0.79,
"instances_seen_from_action": {
"eb1dace786c6": [
-9.73,
0.024
],
"577316c523cb": [
-9.73,
20.11
],
"506b547ba744": [
12.893,
20.111
]
},
"phases": {
"before": {
"seconds": 9.738,
"write": {
"ok": 4162,
"errors": 0,
"ok_rps": 427.4,
"latency_ms": {
"p50": 4.6,
"p95": 7.6,
"p99": 14.6
}
},
"read": {
"ok": 4161,
"errors": 0,
"ok_rps": 427.3,
"latency_ms": {
"p50": 3.6,
"p95": 5.8,
"p99": 11.7
}
}
},
"during": {
"seconds": 12.103,
"write": {
"ok": 3406,
"errors": 1,
"ok_rps": 281.4,
"latency_ms": {
"p50": 6.9,
"p95": 12.7,
"p99": 22.9
}
},
"read": {
"ok": 3404,
"errors": 0,
"ok_rps": 281.3,
"latency_ms": {
"p50": 5.6,
"p95": 10.1,
"p99": 16.4
}
}
},
"after": {
"seconds": 8.008,
"write": {
"ok": 3269,
"errors": 0,
"ok_rps": 408.2,
"latency_ms": {
"p50": 4.7,
"p95": 8.1,
"p99": 21.3
}
},
"read": {
"ok": 3271,
"errors": 0,
"ok_rps": 408.5,
"latency_ms": {
"p50": 3.6,
"p95": 6.1,
"p99": 15.5
}
}
}
},
"recovery": {
"write_errors": 1,
"error_kinds": {
"HTTP 503": 1
},
"write_recovery_seconds": 0.0,
"max_write_gap_seconds": 0.07
},
"integrity": {
"messages_in_history": 10837,
"confirmed_unique": 10837,
"confirmed_missing_or_changed": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"retried_writes_confirmed_as_replay": 0
},
"container_stats_before_action": [
"messenger-lab3-nginx-1\t9.89%\t2.816MiB / 64MiB",
"messenger-lab3-postgres-1\t27.37%\t80.08MiB / 512MiB",
"messenger-lab3-api-1\t84.41%\t48.93MiB / 512MiB",
"messenger-lab3-api-2\t84.27%\t47.33MiB / 512MiB"
],
"action_output": [
"messenger-lab3-api-1"
],
"restore_output_tail": [],
"timeline": [
{
"t": 0,
"ok": 826,
"errors": 0,
"instances": {
"eb1dace786c6": 413,
"577316c523cb": 413
}
},
{
"t": 1,
"ok": 645,
"errors": 0,
"instances": {
"577316c523cb": 322,
"eb1dace786c6": 323
}
},
{
"t": 2,
"ok": 963,
"errors": 0,
"instances": {
"eb1dace786c6": 481,
"577316c523cb": 482
}
},
{
"t": 3,
"ok": 828,
"errors": 0,
"instances": {
"eb1dace786c6": 414,
"577316c523cb": 414
}
},
{
"t": 4,
"ok": 899,
"errors": 0,
"instances": {
"eb1dace786c6": 450,
"577316c523cb": 449
}
},
{
"t": 5,
"ok": 852,
"errors": 0,
"instances": {
"577316c523cb": 426,
"eb1dace786c6": 426
}
},
{
"t": 6,
"ok": 704,
"errors": 0,
"instances": {
"eb1dace786c6": 352,
"577316c523cb": 352
}
},
{
"t": 7,
"ok": 918,
"errors": 0,
"instances": {
"eb1dace786c6": 459,
"577316c523cb": 459
}
},
{
"t": 8,
"ok": 975,
"errors": 0,
"instances": {
"eb1dace786c6": 487,
"577316c523cb": 488
}
},
{
"t": 9,
"ok": 853,
"errors": 1,
"instances": {
"eb1dace786c6": 365,
"577316c523cb": 488
}
},
{
"t": 10,
"ok": 621,
"errors": 0,
"instances": {
"577316c523cb": 621
}
},
{
"t": 11,
"ok": 618,
"errors": 0,
"instances": {
"577316c523cb": 618
}
},
{
"t": 12,
"ok": 474,
"errors": 0,
"instances": {
"577316c523cb": 474
}
},
{
"t": 13,
"ok": 569,
"errors": 0,
"instances": {
"577316c523cb": 569
}
},
{
"t": 14,
"ok": 633,
"errors": 0,
"instances": {
"577316c523cb": 633
}
},
{
"t": 15,
"ok": 548,
"errors": 0,
"instances": {
"577316c523cb": 548
}
},
{
"t": 16,
"ok": 592,
"errors": 0,
"instances": {
"577316c523cb": 592
}
},
{
"t": 17,
"ok": 459,
"errors": 0,
"instances": {
"577316c523cb": 459
}
},
{
"t": 18,
"ok": 570,
"errors": 0,
"instances": {
"577316c523cb": 570
}
},
{
"t": 19,
"ok": 612,
"errors": 0,
"instances": {
"577316c523cb": 612
}
},
{
"t": 20,
"ok": 487,
"errors": 0,
"instances": {
"577316c523cb": 487
}
},
{
"t": 21,
"ok": 580,
"errors": 0,
"instances": {
"577316c523cb": 580
}
},
{
"t": 22,
"ok": 742,
"errors": 0,
"instances": {
"577316c523cb": 568,
"506b547ba744": 174
}
},
{
"t": 23,
"ok": 605,
"errors": 0,
"instances": {
"577316c523cb": 303,
"506b547ba744": 302
}
},
{
"t": 24,
"ok": 959,
"errors": 0,
"instances": {
"577316c523cb": 480,
"506b547ba744": 479
}
},
{
"t": 25,
"ok": 921,
"errors": 0,
"instances": {
"506b547ba744": 461,
"577316c523cb": 460
}
},
{
"t": 26,
"ok": 745,
"errors": 0,
"instances": {
"577316c523cb": 373,
"506b547ba744": 372
}
},
{
"t": 27,
"ok": 889,
"errors": 0,
"instances": {
"506b547ba744": 446,
"577316c523cb": 443
}
},
{
"t": 28,
"ok": 748,
"errors": 0,
"instances": {
"577316c523cb": 375,
"506b547ba744": 373
}
},
{
"t": 29,
"ok": 839,
"errors": 0,
"instances": {
"577316c523cb": 419,
"506b547ba744": 420
}
}
]
},
{
"scenario": "SIGKILL messenger-lab3-api-2",
"target_instance": "577316c523cb",
"workers": 4,
"events_seconds": {
"action_start": 9.946,
"action_end": 10.207,
"restore_start": 20.209,
"restore_end": 22.126,
"end": 30.136
},
"instances_before": [
"506b547ba744",
"577316c523cb"
],
"instances_after": [
"1c20e7f6dbde",
"506b547ba744"
],
"target_responses_while_stopped": 0,
"restored_replica_in_rotation_seconds": 0.0,
"instances_seen_from_action": {
"506b547ba744": [
-9.94,
20.189
],
"577316c523cb": [
-9.939,
0.028
],
"1c20e7f6dbde": [
11.911,
20.19
]
},
"phases": {
"before": {
"seconds": 9.946,
"write": {
"ok": 3853,
"errors": 0,
"ok_rps": 387.4,
"latency_ms": {
"p50": 4.8,
"p95": 9.1,
"p99": 34.8
}
},
"read": {
"ok": 3850,
"errors": 0,
"ok_rps": 387.1,
"latency_ms": {
"p50": 3.7,
"p95": 7.0,
"p99": 19.5
}
}
},
"during": {
"seconds": 12.18,
"write": {
"ok": 3370,
"errors": 3,
"ok_rps": 276.7,
"latency_ms": {
"p50": 6.8,
"p95": 12.8,
"p99": 26.8
}
},
"read": {
"ok": 3372,
"errors": 0,
"ok_rps": 276.8,
"latency_ms": {
"p50": 5.5,
"p95": 11.0,
"p99": 28.6
}
}
},
"after": {
"seconds": 8.01,
"write": {
"ok": 2989,
"errors": 0,
"ok_rps": 373.2,
"latency_ms": {
"p50": 5.0,
"p95": 10.1,
"p99": 35.1
}
},
"read": {
"ok": 2990,
"errors": 0,
"ok_rps": 373.3,
"latency_ms": {
"p50": 3.8,
"p95": 7.6,
"p99": 22.2
}
}
}
},
"recovery": {
"write_errors": 3,
"error_kinds": {
"HTTP 503": 3
},
"write_recovery_seconds": 0.006,
"max_write_gap_seconds": 0.061
},
"integrity": {
"messages_in_history": 10212,
"confirmed_unique": 10212,
"confirmed_missing_or_changed": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"retried_writes_confirmed_as_replay": 0
},
"container_stats_before_action": [
"messenger-lab3-nginx-1\t9.12%\t2.887MiB / 64MiB",
"messenger-lab3-postgres-1\t27.19%\t90.89MiB / 512MiB",
"messenger-lab3-api-1\t80.43%\t47.9MiB / 512MiB",
"messenger-lab3-api-2\t80.26%\t50.77MiB / 512MiB"
],
"action_output": [
"messenger-lab3-api-2"
],
"restore_output_tail": [],
"timeline": [
{
"t": 0,
"ok": 925,
"errors": 0,
"instances": {
"506b547ba744": 462,
"577316c523cb": 463
}
},
{
"t": 1,
"ok": 665,
"errors": 0,
"instances": {
"506b547ba744": 333,
"577316c523cb": 332
}
},
{
"t": 2,
"ok": 856,
"errors": 0,
"instances": {
"577316c523cb": 428,
"506b547ba744": 428
}
},
{
"t": 3,
"ok": 641,
"errors": 0,
"instances": {
"577316c523cb": 321,
"506b547ba744": 320
}
},
{
"t": 4,
"ok": 868,
"errors": 0,
"instances": {
"577316c523cb": 434,
"506b547ba744": 434
}
},
{
"t": 5,
"ok": 874,
"errors": 0,
"instances": {
"506b547ba744": 437,
"577316c523cb": 437
}
},
{
"t": 6,
"ok": 753,
"errors": 0,
"instances": {
"577316c523cb": 378,
"506b547ba744": 375
}
},
{
"t": 7,
"ok": 875,
"errors": 0,
"instances": {
"506b547ba744": 439,
"577316c523cb": 436
}
},
{
"t": 8,
"ok": 649,
"errors": 0,
"instances": {
"506b547ba744": 325,
"577316c523cb": 324
}
},
{
"t": 9,
"ok": 622,
"errors": 3,
"instances": {
"577316c523cb": 309,
"506b547ba744": 313
}
},
{
"t": 10,
"ok": 532,
"errors": 0,
"instances": {
"506b547ba744": 532
}
},
{
"t": 11,
"ok": 653,
"errors": 0,
"instances": {
"506b547ba744": 653
}
},
{
"t": 12,
"ok": 361,
"errors": 0,
"instances": {
"506b547ba744": 361
}
},
{
"t": 13,
"ok": 626,
"errors": 0,
"instances": {
"506b547ba744": 626
}
},
{
"t": 14,
"ok": 618,
"errors": 0,
"instances": {
"506b547ba744": 618
}
},
{
"t": 15,
"ok": 649,
"errors": 0,
"instances": {
"506b547ba744": 649
}
},
{
"t": 16,
"ok": 651,
"errors": 0,
"instances": {
"506b547ba744": 651
}
},
{
"t": 17,
"ok": 461,
"errors": 0,
"instances": {
"506b547ba744": 461
}
},
{
"t": 18,
"ok": 516,
"errors": 0,
"instances": {
"506b547ba744": 516
}
},
{
"t": 19,
"ok": 611,
"errors": 0,
"instances": {
"506b547ba744": 611
}
},
{
"t": 20,
"ok": 436,
"errors": 0,
"instances": {
"506b547ba744": 436
}
},
{
"t": 21,
"ok": 487,
"errors": 0,
"instances": {
"506b547ba744": 451,
"1c20e7f6dbde": 36
}
},
{
"t": 22,
"ok": 769,
"errors": 0,
"instances": {
"1c20e7f6dbde": 385,
"506b547ba744": 384
}
},
{
"t": 23,
"ok": 614,
"errors": 0,
"instances": {
"1c20e7f6dbde": 307,
"506b547ba744": 307
}
},
{
"t": 24,
"ok": 804,
"errors": 0,
"instances": {
"506b547ba744": 402,
"1c20e7f6dbde": 402
}
},
{
"t": 25,
"ok": 717,
"errors": 0,
"instances": {
"1c20e7f6dbde": 358,
"506b547ba744": 359
}
},
{
"t": 26,
"ok": 827,
"errors": 0,
"instances": {
"506b547ba744": 415,
"1c20e7f6dbde": 412
}
},
{
"t": 27,
"ok": 735,
"errors": 0,
"instances": {
"1c20e7f6dbde": 369,
"506b547ba744": 366
}
},
{
"t": 28,
"ok": 648,
"errors": 0,
"instances": {
"1c20e7f6dbde": 326,
"506b547ba744": 322
}
},
{
"t": 29,
"ok": 851,
"errors": 0,
"instances": {
"506b547ba744": 427,
"1c20e7f6dbde": 424
}
},
{
"t": 30,
"ok": 130,
"errors": 0,
"instances": {
"506b547ba744": 65,
"1c20e7f6dbde": 65
}
}
]
}
]
}