Files
mtusi2026lab3/evidence/failover-sequential.json
T
vest1ter cde3374d8a ЛР3: балансировка HAProxy, 2 реплики API, идемпотентность, отказоустойчивость (на 4)
- X-Instance-Id в каждом ответе; обязательный Idempotency-Key, таблица idempotency_keys
  (миграция 0002, одна транзакция, конкурентные повторы через уникальный индекс)
- graceful shutdown: SIGTERM -> ready=503 -> drain 3 с -> закрытие порта
- compose: lb (HAProxy) + api x2 без портов + разовый migrate + postgres, лимиты CPU/RAM
- хуки stop-one/restart/restore-replicas, scripts/experiments.py (failover, idempotency, rolling)
- свои тесты: 44 (13 новых); evidence: smoke, pytest, persistence, failover, эксперименты
2026-10-29 10:58:17 +03:00

713 lines
13 KiB
JSON

{
"method": "closed loop 8 workers: POST message (retry same Idempotency-Key on error) + GET page; client timeout 2s; each replica stopped with docker stop -t 8, restored, then the next",
"replicas": [
"bb53391a07f1",
"e51c99b5436b"
],
"warmup_seconds": 5,
"seconds_down": 15,
"runs": [
{
"stopped_replica": "bb53391a07f1",
"baseline_before_stop": {
"seconds": 7.73,
"successful_requests": 339,
"failed_requests": 0,
"error_reasons": {},
"error_ratio": 0.0,
"successful_rps": 43.8,
"latency_ms": {
"p50": 147.3,
"p95": 398.0,
"p99": 597.8
}
},
"stop_to_restore": {
"seconds": 19.62,
"successful_requests": 467,
"failed_requests": 0,
"error_reasons": {},
"error_ratio": 0.0,
"successful_rps": 23.8,
"latency_ms": {
"p50": 314.1,
"p95": 602.1,
"p99": 699.8
}
},
"after_restore": {
"seconds": 29.0,
"successful_requests": 894,
"failed_requests": 0,
"error_reasons": {},
"error_ratio": 0.0,
"successful_rps": 30.8,
"latency_ms": {
"p50": 253.7,
"p95": 590.4,
"p99": 704.4
}
},
"docker_stop_seconds": 4.62,
"last_error_seconds_after_stop": 0.0,
"first_success_on_other_replica_seconds": 0.02,
"requests_served_by_stopped_replica_after_stop_returned": 0,
"seconds_until_replica_back_in_rotation": 20.86,
"instances_seen": [
"bb53391a07f1",
"e51c99b5436b"
],
"resource_snapshot_under_load": [
"mtusi-lab3-api-1: cpu 90.61%, mem 81.91MiB / 512MiB",
"mtusi-lab3-api-2: cpu 103.60%, mem 82.59MiB / 512MiB",
"mtusi-lab3-lb-1: cpu 15.84%, mem 13.01MiB / 128MiB",
"mtusi-lab3-postgres-1: cpu 57.27%, mem 54.5MiB / 512MiB"
],
"timeline_per_second": [
{
"t": 0,
"ok": 34,
"errors": 0
},
{
"t": 1,
"ok": 45,
"errors": 0
},
{
"t": 2,
"ok": 29,
"errors": 0
},
{
"t": 3,
"ok": 45,
"errors": 0
},
{
"t": 4,
"ok": 50,
"errors": 0
},
{
"t": 5,
"ok": 48,
"errors": 0
},
{
"t": 6,
"ok": 54,
"errors": 0
},
{
"t": 7,
"ok": 48,
"errors": 0
},
{
"t": 8,
"ok": 32,
"errors": 0
},
{
"t": 9,
"ok": 16,
"errors": 0
},
{
"t": 10,
"ok": 23,
"errors": 0
},
{
"t": 11,
"ok": 22,
"errors": 0
},
{
"t": 12,
"ok": 21,
"errors": 0
},
{
"t": 13,
"ok": 26,
"errors": 0
},
{
"t": 14,
"ok": 27,
"errors": 0
},
{
"t": 15,
"ok": 18,
"errors": 0
},
{
"t": 16,
"ok": 15,
"errors": 0
},
{
"t": 17,
"ok": 18,
"errors": 0
},
{
"t": 18,
"ok": 26,
"errors": 0
},
{
"t": 19,
"ok": 30,
"errors": 0
},
{
"t": 20,
"ok": 27,
"errors": 0
},
{
"t": 21,
"ok": 27,
"errors": 0
},
{
"t": 22,
"ok": 29,
"errors": 0
},
{
"t": 23,
"ok": 16,
"errors": 0
},
{
"t": 24,
"ok": 13,
"errors": 0
},
{
"t": 25,
"ok": 28,
"errors": 0
},
{
"t": 26,
"ok": 30,
"errors": 0
},
{
"t": 27,
"ok": 25,
"errors": 0
},
{
"t": 28,
"ok": 26,
"errors": 0
},
{
"t": 29,
"ok": 25,
"errors": 0
},
{
"t": 30,
"ok": 18,
"errors": 0
},
{
"t": 31,
"ok": 15,
"errors": 0
},
{
"t": 32,
"ok": 22,
"errors": 0
},
{
"t": 33,
"ok": 24,
"errors": 0
},
{
"t": 34,
"ok": 30,
"errors": 0
},
{
"t": 35,
"ok": 27,
"errors": 0
},
{
"t": 36,
"ok": 27,
"errors": 0
},
{
"t": 37,
"ok": 19,
"errors": 0
},
{
"t": 38,
"ok": 16,
"errors": 0
},
{
"t": 39,
"ok": 16,
"errors": 0
},
{
"t": 40,
"ok": 25,
"errors": 0
},
{
"t": 41,
"ok": 28,
"errors": 0
},
{
"t": 42,
"ok": 29,
"errors": 0
},
{
"t": 43,
"ok": 25,
"errors": 0
},
{
"t": 44,
"ok": 30,
"errors": 0
},
{
"t": 45,
"ok": 38,
"errors": 0
},
{
"t": 46,
"ok": 22,
"errors": 0
},
{
"t": 47,
"ok": 40,
"errors": 0
},
{
"t": 48,
"ok": 51,
"errors": 0
},
{
"t": 49,
"ok": 56,
"errors": 0
},
{
"t": 50,
"ok": 38,
"errors": 0
},
{
"t": 51,
"ok": 37,
"errors": 0
},
{
"t": 52,
"ok": 49,
"errors": 0
},
{
"t": 53,
"ok": 27,
"errors": 0
},
{
"t": 54,
"ok": 40,
"errors": 0
},
{
"t": 55,
"ok": 56,
"errors": 0
},
{
"t": 56,
"ok": 22,
"errors": 0
}
],
"acked_messages": 850,
"messages_in_history": 850,
"lost_acked_messages": 0,
"duplicate_messages": 0
},
{
"stopped_replica": "e51c99b5436b",
"baseline_before_stop": {
"seconds": 7.62,
"successful_requests": 295,
"failed_requests": 0,
"error_reasons": {},
"error_ratio": 0.0,
"successful_rps": 38.7,
"latency_ms": {
"p50": 169.1,
"p95": 520.0,
"p99": 644.0
}
},
"stop_to_restore": {
"seconds": 19.96,
"successful_requests": 465,
"failed_requests": 0,
"error_reasons": {},
"error_ratio": 0.0,
"successful_rps": 23.3,
"latency_ms": {
"p50": 321.7,
"p95": 602.9,
"p99": 694.0
}
},
"after_restore": {
"seconds": 29.98,
"successful_requests": 966,
"failed_requests": 0,
"error_reasons": {},
"error_ratio": 0.0,
"successful_rps": 32.2,
"latency_ms": {
"p50": 237.7,
"p95": 579.4,
"p99": 767.1
}
},
"docker_stop_seconds": 4.96,
"last_error_seconds_after_stop": 0.0,
"first_success_on_other_replica_seconds": 0.0,
"requests_served_by_stopped_replica_after_stop_returned": 0,
"seconds_until_replica_back_in_rotation": 21.78,
"instances_seen": [
"bb53391a07f1",
"e51c99b5436b"
],
"resource_snapshot_under_load": [
"mtusi-lab3-api-1: cpu 103.71%, mem 82.34MiB / 512MiB",
"mtusi-lab3-api-2: cpu 89.86%, mem 90.22MiB / 512MiB",
"mtusi-lab3-lb-1: cpu 24.15%, mem 13.37MiB / 128MiB",
"mtusi-lab3-postgres-1: cpu 34.28%, mem 54.15MiB / 512MiB"
],
"timeline_per_second": [
{
"t": 0,
"ok": 19,
"errors": 0
},
{
"t": 1,
"ok": 30,
"errors": 0
},
{
"t": 2,
"ok": 53,
"errors": 0
},
{
"t": 3,
"ok": 35,
"errors": 0
},
{
"t": 4,
"ok": 28,
"errors": 0
},
{
"t": 5,
"ok": 53,
"errors": 0
},
{
"t": 6,
"ok": 52,
"errors": 0
},
{
"t": 7,
"ok": 38,
"errors": 0
},
{
"t": 8,
"ok": 28,
"errors": 0
},
{
"t": 9,
"ok": 24,
"errors": 0
},
{
"t": 10,
"ok": 28,
"errors": 0
},
{
"t": 11,
"ok": 27,
"errors": 0
},
{
"t": 12,
"ok": 27,
"errors": 0
},
{
"t": 13,
"ok": 22,
"errors": 0
},
{
"t": 14,
"ok": 19,
"errors": 0
},
{
"t": 15,
"ok": 15,
"errors": 0
},
{
"t": 16,
"ok": 17,
"errors": 0
},
{
"t": 17,
"ok": 27,
"errors": 0
},
{
"t": 18,
"ok": 21,
"errors": 0
},
{
"t": 19,
"ok": 27,
"errors": 0
},
{
"t": 20,
"ok": 27,
"errors": 0
},
{
"t": 21,
"ok": 24,
"errors": 0
},
{
"t": 22,
"ok": 13,
"errors": 0
},
{
"t": 23,
"ok": 13,
"errors": 0
},
{
"t": 24,
"ok": 21,
"errors": 0
},
{
"t": 25,
"ok": 26,
"errors": 0
},
{
"t": 26,
"ok": 30,
"errors": 0
},
{
"t": 27,
"ok": 27,
"errors": 0
},
{
"t": 28,
"ok": 24,
"errors": 0
},
{
"t": 29,
"ok": 20,
"errors": 0
},
{
"t": 30,
"ok": 15,
"errors": 0
},
{
"t": 31,
"ok": 16,
"errors": 0
},
{
"t": 32,
"ok": 22,
"errors": 0
},
{
"t": 33,
"ok": 24,
"errors": 0
},
{
"t": 34,
"ok": 25,
"errors": 0
},
{
"t": 35,
"ok": 28,
"errors": 0
},
{
"t": 36,
"ok": 21,
"errors": 0
},
{
"t": 37,
"ok": 13,
"errors": 0
},
{
"t": 38,
"ok": 13,
"errors": 0
},
{
"t": 39,
"ok": 16,
"errors": 0
},
{
"t": 40,
"ok": 29,
"errors": 0
},
{
"t": 41,
"ok": 28,
"errors": 0
},
{
"t": 42,
"ok": 24,
"errors": 0
},
{
"t": 43,
"ok": 28,
"errors": 0
},
{
"t": 44,
"ok": 29,
"errors": 0
},
{
"t": 45,
"ok": 25,
"errors": 0
},
{
"t": 46,
"ok": 39,
"errors": 0
},
{
"t": 47,
"ok": 49,
"errors": 0
},
{
"t": 48,
"ok": 51,
"errors": 0
},
{
"t": 49,
"ok": 57,
"errors": 0
},
{
"t": 50,
"ok": 50,
"errors": 0
},
{
"t": 51,
"ok": 34,
"errors": 0
},
{
"t": 52,
"ok": 31,
"errors": 0
},
{
"t": 53,
"ok": 47,
"errors": 0
},
{
"t": 54,
"ok": 57,
"errors": 0
},
{
"t": 55,
"ok": 55,
"errors": 0
},
{
"t": 56,
"ok": 54,
"errors": 0
},
{
"t": 57,
"ok": 31,
"errors": 0
}
],
"acked_messages": 863,
"messages_in_history": 1713,
"lost_acked_messages": 0,
"duplicate_messages": 0
}
]
}