forked from mtusi2026/mtusi2026lab3
Второй прогон scripts/collect_evidence.py на новом clone (коммит 2708732):
make check/validate/test, test-persistence (новые PID у lb, api x2,
postgres, valkey), test-failover для обеих реплик по очереди (0 неудачных
попыток), 201 свой тест, серия отказов stop/kill/pause x 2 реплики,
rolling update под нагрузкой (0 ошибок), tests/load.py. Таблица
критерий -> команда -> результат -> файл, раздел уровня 4 с периодами
проверок и временем обнаружения.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
505 lines
12 KiB
JSON
505 lines
12 KiB
JSON
{
|
|
"meta": {
|
|
"date": "2026-10-19T15:23:54+00:00",
|
|
"commit": "2708732",
|
|
"host": "Windows 10, docker 29.8.0",
|
|
"workload": "4 closed-loop workers from the host via 127.0.0.1:8080: POST message (Idempotency-Key, uncertain -> same key) + GET last page; client timeout 3 s",
|
|
"lb": "HAProxy: check GET /health/ready inter 1s fastinter 500ms fall 2 rise 2, timeout check 1s; retry-on conn-failure + redispatch",
|
|
"api": "DRAIN_SECONDS=3, SHUTDOWN_TIMEOUT_SECONDS=8, stop_grace_period 15s"
|
|
},
|
|
"rounds": [
|
|
{
|
|
"mode": "stop",
|
|
"replica": "mtusi-lab3-api-1",
|
|
"instance_id": "907f96037a59",
|
|
"stop_command_seconds": 4.03,
|
|
"before": {
|
|
"seconds": 5.5,
|
|
"requests": 1621,
|
|
"ok": 1621,
|
|
"failed": 0,
|
|
"ok_rps": 294.6,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"201": 812,
|
|
"200": 809
|
|
},
|
|
"instances": {
|
|
"c3f64421973d": 810,
|
|
"907f96037a59": 811
|
|
},
|
|
"p50_ms": 10.88,
|
|
"p95_ms": 31.93,
|
|
"p99_ms": 60.82
|
|
},
|
|
"during": {
|
|
"seconds": 16.03,
|
|
"requests": 3568,
|
|
"ok": 3568,
|
|
"failed": 0,
|
|
"ok_rps": 222.5,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"200": 1784,
|
|
"201": 1784
|
|
},
|
|
"instances": {
|
|
"c3f64421973d": 3434,
|
|
"907f96037a59": 134
|
|
},
|
|
"p50_ms": 13.82,
|
|
"p95_ms": 61.1,
|
|
"p99_ms": 71.93
|
|
},
|
|
"after": {
|
|
"seconds": 6.02,
|
|
"requests": 1503,
|
|
"ok": 1503,
|
|
"failed": 0,
|
|
"ok_rps": 249.7,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"201": 750,
|
|
"200": 753
|
|
},
|
|
"instances": {
|
|
"c3f64421973d": 753,
|
|
"907f96037a59": 750
|
|
},
|
|
"p50_ms": 11.19,
|
|
"p95_ms": 55.98,
|
|
"p99_ms": 63.38
|
|
},
|
|
"failed_after_stop": 0,
|
|
"failed_statuses": [],
|
|
"last_failure_after_stop_s": 0.0,
|
|
"target_last_served_after_stop_s": 1.283,
|
|
"rejoin_up_in_lb_s": 3.25,
|
|
"rejoin_serving_s": 3.35,
|
|
"client_retries_same_key": 0,
|
|
"history": {
|
|
"history_messages": 3692,
|
|
"confirmed_messages": 3692,
|
|
"lost_confirmed": 0,
|
|
"altered_confirmed": 0,
|
|
"duplicate_ids": 0,
|
|
"duplicate_texts": 0,
|
|
"unconfirmed_but_stored": 0
|
|
},
|
|
"restarted_state": "running health=healthy"
|
|
},
|
|
{
|
|
"mode": "stop",
|
|
"replica": "mtusi-lab3-api-2",
|
|
"instance_id": "c3f64421973d",
|
|
"stop_command_seconds": 3.79,
|
|
"before": {
|
|
"seconds": 5.5,
|
|
"requests": 1426,
|
|
"ok": 1426,
|
|
"failed": 0,
|
|
"ok_rps": 259.2,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"201": 714,
|
|
"200": 712
|
|
},
|
|
"instances": {
|
|
"c3f64421973d": 713,
|
|
"907f96037a59": 713
|
|
},
|
|
"p50_ms": 11.18,
|
|
"p95_ms": 53.47,
|
|
"p99_ms": 64.41
|
|
},
|
|
"during": {
|
|
"seconds": 15.79,
|
|
"requests": 3565,
|
|
"ok": 3565,
|
|
"failed": 0,
|
|
"ok_rps": 225.7,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"200": 1782,
|
|
"201": 1783
|
|
},
|
|
"instances": {
|
|
"c3f64421973d": 222,
|
|
"907f96037a59": 3343
|
|
},
|
|
"p50_ms": 13.63,
|
|
"p95_ms": 63.19,
|
|
"p99_ms": 71.39
|
|
},
|
|
"after": {
|
|
"seconds": 6.02,
|
|
"requests": 1656,
|
|
"ok": 1656,
|
|
"failed": 0,
|
|
"ok_rps": 275.1,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"201": 827,
|
|
"200": 829
|
|
},
|
|
"instances": {
|
|
"c3f64421973d": 828,
|
|
"907f96037a59": 828
|
|
},
|
|
"p50_ms": 11.19,
|
|
"p95_ms": 41.65,
|
|
"p99_ms": 60.73
|
|
},
|
|
"failed_after_stop": 0,
|
|
"failed_statuses": [],
|
|
"last_failure_after_stop_s": 0.0,
|
|
"target_last_served_after_stop_s": 1.235,
|
|
"rejoin_up_in_lb_s": 1.35,
|
|
"rejoin_serving_s": 3.07,
|
|
"client_retries_same_key": 0,
|
|
"history": {
|
|
"history_messages": 3690,
|
|
"confirmed_messages": 3690,
|
|
"lost_confirmed": 0,
|
|
"altered_confirmed": 0,
|
|
"duplicate_ids": 0,
|
|
"duplicate_texts": 0,
|
|
"unconfirmed_but_stored": 0
|
|
},
|
|
"restarted_state": "running health=healthy"
|
|
},
|
|
{
|
|
"mode": "kill",
|
|
"replica": "mtusi-lab3-api-1",
|
|
"instance_id": "907f96037a59",
|
|
"stop_command_seconds": 0.72,
|
|
"before": {
|
|
"seconds": 5.5,
|
|
"requests": 1491,
|
|
"ok": 1491,
|
|
"failed": 0,
|
|
"ok_rps": 270.9,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"201": 745,
|
|
"200": 746
|
|
},
|
|
"instances": {
|
|
"907f96037a59": 746,
|
|
"c3f64421973d": 745
|
|
},
|
|
"p50_ms": 11.18,
|
|
"p95_ms": 41.28,
|
|
"p99_ms": 56.2
|
|
},
|
|
"during": {
|
|
"seconds": 12.72,
|
|
"requests": 2734,
|
|
"ok": 2731,
|
|
"failed": 3,
|
|
"ok_rps": 214.7,
|
|
"error_ratio": 0.0011,
|
|
"statuses": {
|
|
"201": 1366,
|
|
"200": 1365,
|
|
"0": 3
|
|
},
|
|
"instances": {
|
|
"907f96037a59": 22,
|
|
"c3f64421973d": 2709
|
|
},
|
|
"p50_ms": 13.72,
|
|
"p95_ms": 59.09,
|
|
"p99_ms": 69.77
|
|
},
|
|
"after": {
|
|
"seconds": 6.02,
|
|
"requests": 1477,
|
|
"ok": 1477,
|
|
"failed": 0,
|
|
"ok_rps": 245.5,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"200": 740,
|
|
"201": 737
|
|
},
|
|
"instances": {
|
|
"c3f64421973d": 739,
|
|
"907f96037a59": 738
|
|
},
|
|
"p50_ms": 11.54,
|
|
"p95_ms": 54.73,
|
|
"p99_ms": 62.68
|
|
},
|
|
"failed_after_stop": 3,
|
|
"failed_statuses": [
|
|
0
|
|
],
|
|
"last_failure_after_stop_s": 0.247,
|
|
"target_last_served_after_stop_s": 0.22,
|
|
"rejoin_up_in_lb_s": 1.17,
|
|
"rejoin_serving_s": 2.8,
|
|
"client_retries_same_key": 2,
|
|
"history": {
|
|
"history_messages": 3200,
|
|
"confirmed_messages": 3200,
|
|
"lost_confirmed": 0,
|
|
"altered_confirmed": 0,
|
|
"duplicate_ids": 0,
|
|
"duplicate_texts": 0,
|
|
"unconfirmed_but_stored": 0
|
|
},
|
|
"restarted_state": "running health=healthy"
|
|
},
|
|
{
|
|
"mode": "kill",
|
|
"replica": "mtusi-lab3-api-2",
|
|
"instance_id": "c3f64421973d",
|
|
"stop_command_seconds": 0.77,
|
|
"before": {
|
|
"seconds": 5.5,
|
|
"requests": 1241,
|
|
"ok": 1241,
|
|
"failed": 0,
|
|
"ok_rps": 225.6,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"201": 620,
|
|
"200": 621
|
|
},
|
|
"instances": {
|
|
"c3f64421973d": 621,
|
|
"907f96037a59": 620
|
|
},
|
|
"p50_ms": 11.41,
|
|
"p95_ms": 57.52,
|
|
"p99_ms": 66.91
|
|
},
|
|
"during": {
|
|
"seconds": 12.77,
|
|
"requests": 2297,
|
|
"ok": 2294,
|
|
"failed": 3,
|
|
"ok_rps": 179.6,
|
|
"error_ratio": 0.00131,
|
|
"statuses": {
|
|
"200": 1147,
|
|
"201": 1147,
|
|
"0": 3
|
|
},
|
|
"instances": {
|
|
"907f96037a59": 2276,
|
|
"c3f64421973d": 18
|
|
},
|
|
"p50_ms": 15.06,
|
|
"p95_ms": 66.96,
|
|
"p99_ms": 77.44
|
|
},
|
|
"after": {
|
|
"seconds": 6.01,
|
|
"requests": 1442,
|
|
"ok": 1442,
|
|
"failed": 0,
|
|
"ok_rps": 240.0,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"200": 722,
|
|
"201": 720
|
|
},
|
|
"instances": {
|
|
"c3f64421973d": 721,
|
|
"907f96037a59": 721
|
|
},
|
|
"p50_ms": 11.64,
|
|
"p95_ms": 53.4,
|
|
"p99_ms": 65.32
|
|
},
|
|
"failed_after_stop": 3,
|
|
"failed_statuses": [
|
|
0
|
|
],
|
|
"last_failure_after_stop_s": 0.214,
|
|
"target_last_served_after_stop_s": 0.174,
|
|
"rejoin_up_in_lb_s": 2.32,
|
|
"rejoin_serving_s": 3.64,
|
|
"client_retries_same_key": 2,
|
|
"history": {
|
|
"history_messages": 2885,
|
|
"confirmed_messages": 2885,
|
|
"lost_confirmed": 0,
|
|
"altered_confirmed": 0,
|
|
"duplicate_ids": 0,
|
|
"duplicate_texts": 0,
|
|
"unconfirmed_but_stored": 0
|
|
},
|
|
"restarted_state": "running health=healthy"
|
|
},
|
|
{
|
|
"mode": "pause",
|
|
"replica": "mtusi-lab3-api-1",
|
|
"instance_id": "907f96037a59",
|
|
"stop_command_seconds": 0.2,
|
|
"before": {
|
|
"seconds": 5.5,
|
|
"requests": 1493,
|
|
"ok": 1493,
|
|
"failed": 0,
|
|
"ok_rps": 271.3,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"200": 747,
|
|
"201": 746
|
|
},
|
|
"instances": {
|
|
"907f96037a59": 747,
|
|
"c3f64421973d": 746
|
|
},
|
|
"p50_ms": 11.06,
|
|
"p95_ms": 49.85,
|
|
"p99_ms": 61.7
|
|
},
|
|
"during": {
|
|
"seconds": 12.2,
|
|
"requests": 1653,
|
|
"ok": 1649,
|
|
"failed": 4,
|
|
"ok_rps": 135.2,
|
|
"error_ratio": 0.00242,
|
|
"statuses": {
|
|
"201": 824,
|
|
"200": 825,
|
|
"0": 4
|
|
},
|
|
"instances": {
|
|
"c3f64421973d": 1637,
|
|
"907f96037a59": 12
|
|
},
|
|
"p50_ms": 15.99,
|
|
"p95_ms": 68.99,
|
|
"p99_ms": 79.54
|
|
},
|
|
"after": {
|
|
"seconds": 6.03,
|
|
"requests": 1474,
|
|
"ok": 1474,
|
|
"failed": 0,
|
|
"ok_rps": 244.6,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"201": 736,
|
|
"200": 738
|
|
},
|
|
"instances": {
|
|
"907f96037a59": 737,
|
|
"c3f64421973d": 737
|
|
},
|
|
"p50_ms": 12.41,
|
|
"p95_ms": 49.6,
|
|
"p99_ms": 64.12
|
|
},
|
|
"failed_after_stop": 4,
|
|
"failed_statuses": [
|
|
0
|
|
],
|
|
"last_failure_after_stop_s": 3.204,
|
|
"target_last_served_after_stop_s": 0.148,
|
|
"rejoin_up_in_lb_s": 2.21,
|
|
"rejoin_serving_s": 2.31,
|
|
"client_retries_same_key": 3,
|
|
"history": {
|
|
"history_messages": 2610,
|
|
"confirmed_messages": 2610,
|
|
"lost_confirmed": 0,
|
|
"altered_confirmed": 0,
|
|
"duplicate_ids": 0,
|
|
"duplicate_texts": 0,
|
|
"unconfirmed_but_stored": 0
|
|
},
|
|
"restarted_state": "running health=healthy"
|
|
},
|
|
{
|
|
"mode": "pause",
|
|
"replica": "mtusi-lab3-api-2",
|
|
"instance_id": "c3f64421973d",
|
|
"stop_command_seconds": 0.13,
|
|
"before": {
|
|
"seconds": 5.5,
|
|
"requests": 1395,
|
|
"ok": 1395,
|
|
"failed": 0,
|
|
"ok_rps": 253.6,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"200": 697,
|
|
"201": 698
|
|
},
|
|
"instances": {
|
|
"c3f64421973d": 698,
|
|
"907f96037a59": 697
|
|
},
|
|
"p50_ms": 12.39,
|
|
"p95_ms": 44.63,
|
|
"p99_ms": 60.81
|
|
},
|
|
"during": {
|
|
"seconds": 12.13,
|
|
"requests": 1787,
|
|
"ok": 1783,
|
|
"failed": 4,
|
|
"ok_rps": 147.0,
|
|
"error_ratio": 0.00224,
|
|
"statuses": {
|
|
"200": 890,
|
|
"201": 893,
|
|
"0": 4
|
|
},
|
|
"instances": {
|
|
"907f96037a59": 1768,
|
|
"c3f64421973d": 15
|
|
},
|
|
"p50_ms": 15.86,
|
|
"p95_ms": 65.74,
|
|
"p99_ms": 73.59
|
|
},
|
|
"after": {
|
|
"seconds": 6.02,
|
|
"requests": 1522,
|
|
"ok": 1522,
|
|
"failed": 0,
|
|
"ok_rps": 253.0,
|
|
"error_ratio": 0.0,
|
|
"statuses": {
|
|
"200": 762,
|
|
"201": 760
|
|
},
|
|
"instances": {
|
|
"c3f64421973d": 761,
|
|
"907f96037a59": 761
|
|
},
|
|
"p50_ms": 12.2,
|
|
"p95_ms": 44.57,
|
|
"p99_ms": 61.5
|
|
},
|
|
"failed_after_stop": 4,
|
|
"failed_statuses": [
|
|
0
|
|
],
|
|
"last_failure_after_stop_s": 3.12,
|
|
"target_last_served_after_stop_s": 0.098,
|
|
"rejoin_up_in_lb_s": 1.16,
|
|
"rejoin_serving_s": 1.27,
|
|
"client_retries_same_key": 1,
|
|
"history": {
|
|
"history_messages": 2558,
|
|
"confirmed_messages": 2558,
|
|
"lost_confirmed": 0,
|
|
"altered_confirmed": 0,
|
|
"duplicate_ids": 0,
|
|
"duplicate_texts": 0,
|
|
"unconfirmed_but_stored": 0
|
|
},
|
|
"restarted_state": "running health=healthy"
|
|
}
|
|
]
|
|
}
|