Files
mtusi2026lab3/evidence/failover/series.json
T
Alexey BodanyukandClaude Opus 5.5 c758e49cbb docs: REPORT lab3 — доказательства со стенда из чистого clone, уровень 4, трек 1
Второй прогон scripts/collect_evidence.py на новом clone (коммит 2708732):
make check/validate/test, test-persistence (новые PID у lb, api x2,
postgres, valkey), test-failover для обеих реплик по очереди (0 неудачных
попыток), 201 свой тест, серия отказов stop/kill/pause x 2 реплики,
rolling update под нагрузкой (0 ошибок), tests/load.py. Таблица
критерий -> команда -> результат -> файл, раздел уровня 4 с периодами
проверок и временем обнаружения.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 18:28:25 +03:00

505 lines
12 KiB
JSON

{
"meta": {
"date": "2026-10-19T15:23:54+00:00",
"commit": "2708732",
"host": "Windows 10, docker 29.8.0",
"workload": "4 closed-loop workers from the host via 127.0.0.1:8080: POST message (Idempotency-Key, uncertain -> same key) + GET last page; client timeout 3 s",
"lb": "HAProxy: check GET /health/ready inter 1s fastinter 500ms fall 2 rise 2, timeout check 1s; retry-on conn-failure + redispatch",
"api": "DRAIN_SECONDS=3, SHUTDOWN_TIMEOUT_SECONDS=8, stop_grace_period 15s"
},
"rounds": [
{
"mode": "stop",
"replica": "mtusi-lab3-api-1",
"instance_id": "907f96037a59",
"stop_command_seconds": 4.03,
"before": {
"seconds": 5.5,
"requests": 1621,
"ok": 1621,
"failed": 0,
"ok_rps": 294.6,
"error_ratio": 0.0,
"statuses": {
"201": 812,
"200": 809
},
"instances": {
"c3f64421973d": 810,
"907f96037a59": 811
},
"p50_ms": 10.88,
"p95_ms": 31.93,
"p99_ms": 60.82
},
"during": {
"seconds": 16.03,
"requests": 3568,
"ok": 3568,
"failed": 0,
"ok_rps": 222.5,
"error_ratio": 0.0,
"statuses": {
"200": 1784,
"201": 1784
},
"instances": {
"c3f64421973d": 3434,
"907f96037a59": 134
},
"p50_ms": 13.82,
"p95_ms": 61.1,
"p99_ms": 71.93
},
"after": {
"seconds": 6.02,
"requests": 1503,
"ok": 1503,
"failed": 0,
"ok_rps": 249.7,
"error_ratio": 0.0,
"statuses": {
"201": 750,
"200": 753
},
"instances": {
"c3f64421973d": 753,
"907f96037a59": 750
},
"p50_ms": 11.19,
"p95_ms": 55.98,
"p99_ms": 63.38
},
"failed_after_stop": 0,
"failed_statuses": [],
"last_failure_after_stop_s": 0.0,
"target_last_served_after_stop_s": 1.283,
"rejoin_up_in_lb_s": 3.25,
"rejoin_serving_s": 3.35,
"client_retries_same_key": 0,
"history": {
"history_messages": 3692,
"confirmed_messages": 3692,
"lost_confirmed": 0,
"altered_confirmed": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"unconfirmed_but_stored": 0
},
"restarted_state": "running health=healthy"
},
{
"mode": "stop",
"replica": "mtusi-lab3-api-2",
"instance_id": "c3f64421973d",
"stop_command_seconds": 3.79,
"before": {
"seconds": 5.5,
"requests": 1426,
"ok": 1426,
"failed": 0,
"ok_rps": 259.2,
"error_ratio": 0.0,
"statuses": {
"201": 714,
"200": 712
},
"instances": {
"c3f64421973d": 713,
"907f96037a59": 713
},
"p50_ms": 11.18,
"p95_ms": 53.47,
"p99_ms": 64.41
},
"during": {
"seconds": 15.79,
"requests": 3565,
"ok": 3565,
"failed": 0,
"ok_rps": 225.7,
"error_ratio": 0.0,
"statuses": {
"200": 1782,
"201": 1783
},
"instances": {
"c3f64421973d": 222,
"907f96037a59": 3343
},
"p50_ms": 13.63,
"p95_ms": 63.19,
"p99_ms": 71.39
},
"after": {
"seconds": 6.02,
"requests": 1656,
"ok": 1656,
"failed": 0,
"ok_rps": 275.1,
"error_ratio": 0.0,
"statuses": {
"201": 827,
"200": 829
},
"instances": {
"c3f64421973d": 828,
"907f96037a59": 828
},
"p50_ms": 11.19,
"p95_ms": 41.65,
"p99_ms": 60.73
},
"failed_after_stop": 0,
"failed_statuses": [],
"last_failure_after_stop_s": 0.0,
"target_last_served_after_stop_s": 1.235,
"rejoin_up_in_lb_s": 1.35,
"rejoin_serving_s": 3.07,
"client_retries_same_key": 0,
"history": {
"history_messages": 3690,
"confirmed_messages": 3690,
"lost_confirmed": 0,
"altered_confirmed": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"unconfirmed_but_stored": 0
},
"restarted_state": "running health=healthy"
},
{
"mode": "kill",
"replica": "mtusi-lab3-api-1",
"instance_id": "907f96037a59",
"stop_command_seconds": 0.72,
"before": {
"seconds": 5.5,
"requests": 1491,
"ok": 1491,
"failed": 0,
"ok_rps": 270.9,
"error_ratio": 0.0,
"statuses": {
"201": 745,
"200": 746
},
"instances": {
"907f96037a59": 746,
"c3f64421973d": 745
},
"p50_ms": 11.18,
"p95_ms": 41.28,
"p99_ms": 56.2
},
"during": {
"seconds": 12.72,
"requests": 2734,
"ok": 2731,
"failed": 3,
"ok_rps": 214.7,
"error_ratio": 0.0011,
"statuses": {
"201": 1366,
"200": 1365,
"0": 3
},
"instances": {
"907f96037a59": 22,
"c3f64421973d": 2709
},
"p50_ms": 13.72,
"p95_ms": 59.09,
"p99_ms": 69.77
},
"after": {
"seconds": 6.02,
"requests": 1477,
"ok": 1477,
"failed": 0,
"ok_rps": 245.5,
"error_ratio": 0.0,
"statuses": {
"200": 740,
"201": 737
},
"instances": {
"c3f64421973d": 739,
"907f96037a59": 738
},
"p50_ms": 11.54,
"p95_ms": 54.73,
"p99_ms": 62.68
},
"failed_after_stop": 3,
"failed_statuses": [
0
],
"last_failure_after_stop_s": 0.247,
"target_last_served_after_stop_s": 0.22,
"rejoin_up_in_lb_s": 1.17,
"rejoin_serving_s": 2.8,
"client_retries_same_key": 2,
"history": {
"history_messages": 3200,
"confirmed_messages": 3200,
"lost_confirmed": 0,
"altered_confirmed": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"unconfirmed_but_stored": 0
},
"restarted_state": "running health=healthy"
},
{
"mode": "kill",
"replica": "mtusi-lab3-api-2",
"instance_id": "c3f64421973d",
"stop_command_seconds": 0.77,
"before": {
"seconds": 5.5,
"requests": 1241,
"ok": 1241,
"failed": 0,
"ok_rps": 225.6,
"error_ratio": 0.0,
"statuses": {
"201": 620,
"200": 621
},
"instances": {
"c3f64421973d": 621,
"907f96037a59": 620
},
"p50_ms": 11.41,
"p95_ms": 57.52,
"p99_ms": 66.91
},
"during": {
"seconds": 12.77,
"requests": 2297,
"ok": 2294,
"failed": 3,
"ok_rps": 179.6,
"error_ratio": 0.00131,
"statuses": {
"200": 1147,
"201": 1147,
"0": 3
},
"instances": {
"907f96037a59": 2276,
"c3f64421973d": 18
},
"p50_ms": 15.06,
"p95_ms": 66.96,
"p99_ms": 77.44
},
"after": {
"seconds": 6.01,
"requests": 1442,
"ok": 1442,
"failed": 0,
"ok_rps": 240.0,
"error_ratio": 0.0,
"statuses": {
"200": 722,
"201": 720
},
"instances": {
"c3f64421973d": 721,
"907f96037a59": 721
},
"p50_ms": 11.64,
"p95_ms": 53.4,
"p99_ms": 65.32
},
"failed_after_stop": 3,
"failed_statuses": [
0
],
"last_failure_after_stop_s": 0.214,
"target_last_served_after_stop_s": 0.174,
"rejoin_up_in_lb_s": 2.32,
"rejoin_serving_s": 3.64,
"client_retries_same_key": 2,
"history": {
"history_messages": 2885,
"confirmed_messages": 2885,
"lost_confirmed": 0,
"altered_confirmed": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"unconfirmed_but_stored": 0
},
"restarted_state": "running health=healthy"
},
{
"mode": "pause",
"replica": "mtusi-lab3-api-1",
"instance_id": "907f96037a59",
"stop_command_seconds": 0.2,
"before": {
"seconds": 5.5,
"requests": 1493,
"ok": 1493,
"failed": 0,
"ok_rps": 271.3,
"error_ratio": 0.0,
"statuses": {
"200": 747,
"201": 746
},
"instances": {
"907f96037a59": 747,
"c3f64421973d": 746
},
"p50_ms": 11.06,
"p95_ms": 49.85,
"p99_ms": 61.7
},
"during": {
"seconds": 12.2,
"requests": 1653,
"ok": 1649,
"failed": 4,
"ok_rps": 135.2,
"error_ratio": 0.00242,
"statuses": {
"201": 824,
"200": 825,
"0": 4
},
"instances": {
"c3f64421973d": 1637,
"907f96037a59": 12
},
"p50_ms": 15.99,
"p95_ms": 68.99,
"p99_ms": 79.54
},
"after": {
"seconds": 6.03,
"requests": 1474,
"ok": 1474,
"failed": 0,
"ok_rps": 244.6,
"error_ratio": 0.0,
"statuses": {
"201": 736,
"200": 738
},
"instances": {
"907f96037a59": 737,
"c3f64421973d": 737
},
"p50_ms": 12.41,
"p95_ms": 49.6,
"p99_ms": 64.12
},
"failed_after_stop": 4,
"failed_statuses": [
0
],
"last_failure_after_stop_s": 3.204,
"target_last_served_after_stop_s": 0.148,
"rejoin_up_in_lb_s": 2.21,
"rejoin_serving_s": 2.31,
"client_retries_same_key": 3,
"history": {
"history_messages": 2610,
"confirmed_messages": 2610,
"lost_confirmed": 0,
"altered_confirmed": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"unconfirmed_but_stored": 0
},
"restarted_state": "running health=healthy"
},
{
"mode": "pause",
"replica": "mtusi-lab3-api-2",
"instance_id": "c3f64421973d",
"stop_command_seconds": 0.13,
"before": {
"seconds": 5.5,
"requests": 1395,
"ok": 1395,
"failed": 0,
"ok_rps": 253.6,
"error_ratio": 0.0,
"statuses": {
"200": 697,
"201": 698
},
"instances": {
"c3f64421973d": 698,
"907f96037a59": 697
},
"p50_ms": 12.39,
"p95_ms": 44.63,
"p99_ms": 60.81
},
"during": {
"seconds": 12.13,
"requests": 1787,
"ok": 1783,
"failed": 4,
"ok_rps": 147.0,
"error_ratio": 0.00224,
"statuses": {
"200": 890,
"201": 893,
"0": 4
},
"instances": {
"907f96037a59": 1768,
"c3f64421973d": 15
},
"p50_ms": 15.86,
"p95_ms": 65.74,
"p99_ms": 73.59
},
"after": {
"seconds": 6.02,
"requests": 1522,
"ok": 1522,
"failed": 0,
"ok_rps": 253.0,
"error_ratio": 0.0,
"statuses": {
"200": 762,
"201": 760
},
"instances": {
"c3f64421973d": 761,
"907f96037a59": 761
},
"p50_ms": 12.2,
"p95_ms": 44.57,
"p99_ms": 61.5
},
"failed_after_stop": 4,
"failed_statuses": [
0
],
"last_failure_after_stop_s": 3.12,
"target_last_served_after_stop_s": 0.098,
"rejoin_up_in_lb_s": 1.16,
"rejoin_serving_s": 1.27,
"client_retries_same_key": 1,
"history": {
"history_messages": 2558,
"confirmed_messages": 2558,
"lost_confirmed": 0,
"altered_confirmed": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"unconfirmed_but_stored": 0
},
"restarted_state": "running health=healthy"
}
]
}