Files
mtusi2026lab3/evidence/failover-sequential-repeat.json
T
vest1ter cde3374d8a ЛР3: балансировка HAProxy, 2 реплики API, идемпотентность, отказоустойчивость (на 4)
- X-Instance-Id в каждом ответе; обязательный Idempotency-Key, таблица idempotency_keys
  (миграция 0002, одна транзакция, конкурентные повторы через уникальный индекс)
- graceful shutdown: SIGTERM -> ready=503 -> drain 3 с -> закрытие порта
- compose: lb (HAProxy) + api x2 без портов + разовый migrate + postgres, лимиты CPU/RAM
- хуки stop-one/restart/restore-replicas, scripts/experiments.py (failover, idempotency, rolling)
- свои тесты: 44 (13 новых); evidence: smoke, pytest, persistence, failover, эксперименты
2026-10-29 10:58:17 +03:00

703 lines
13 KiB
JSON

{
"method": "closed loop 8 workers: POST message (retry same Idempotency-Key on error) + GET page; client timeout 2s; each replica stopped with docker stop -t 8, restored, then the next",
"replicas": [
"3de2fdb570eb",
"945116f913cc"
],
"warmup_seconds": 5,
"seconds_down": 15,
"runs": [
{
"stopped_replica": "3de2fdb570eb",
"baseline_before_stop": {
"seconds": 7.72,
"successful_requests": 359,
"failed_requests": 0,
"error_reasons": {},
"error_ratio": 0.0,
"successful_rps": 46.5,
"latency_ms": {
"p50": 136.4,
"p95": 433.5,
"p99": 628.8
}
},
"stop_to_restore": {
"seconds": 19.38,
"successful_requests": 538,
"failed_requests": 0,
"error_reasons": {},
"error_ratio": 0.0,
"successful_rps": 27.8,
"latency_ms": {
"p50": 281.4,
"p95": 515.9,
"p99": 694.2
}
},
"after_restore": {
"seconds": 28.26,
"successful_requests": 925,
"failed_requests": 0,
"error_reasons": {},
"error_ratio": 0.0,
"successful_rps": 32.7,
"latency_ms": {
"p50": 244.6,
"p95": 537.0,
"p99": 698.6
}
},
"docker_stop_seconds": 4.38,
"last_error_seconds_after_stop": 0.0,
"first_success_on_other_replica_seconds": 0.03,
"requests_served_by_stopped_replica_after_stop_returned": 0,
"seconds_until_replica_back_in_rotation": 20.11,
"instances_seen": [
"3de2fdb570eb",
"945116f913cc"
],
"resource_snapshot_under_load": [
"mtusi-lab3-api-5: cpu 99.82%, mem 80.75MiB / 512MiB",
"mtusi-lab3-api-6: cpu 46.77%, mem 81.65MiB / 512MiB",
"mtusi-lab3-lb-1: cpu 7.09%, mem 17.03MiB / 128MiB",
"mtusi-lab3-postgres-1: cpu 25.45%, mem 64.93MiB / 512MiB"
],
"timeline_per_second": [
{
"t": 0,
"ok": 30,
"errors": 0
},
{
"t": 1,
"ok": 58,
"errors": 0
},
{
"t": 2,
"ok": 60,
"errors": 0
},
{
"t": 3,
"ok": 59,
"errors": 0
},
{
"t": 4,
"ok": 60,
"errors": 0
},
{
"t": 5,
"ok": 35,
"errors": 0
},
{
"t": 6,
"ok": 33,
"errors": 0
},
{
"t": 7,
"ok": 41,
"errors": 0
},
{
"t": 8,
"ok": 56,
"errors": 0
},
{
"t": 9,
"ok": 38,
"errors": 0
},
{
"t": 10,
"ok": 29,
"errors": 0
},
{
"t": 11,
"ok": 29,
"errors": 0
},
{
"t": 12,
"ok": 23,
"errors": 0
},
{
"t": 13,
"ok": 17,
"errors": 0
},
{
"t": 14,
"ok": 15,
"errors": 0
},
{
"t": 15,
"ok": 29,
"errors": 0
},
{
"t": 16,
"ok": 29,
"errors": 0
},
{
"t": 17,
"ok": 31,
"errors": 0
},
{
"t": 18,
"ok": 26,
"errors": 0
},
{
"t": 19,
"ok": 28,
"errors": 0
},
{
"t": 20,
"ok": 17,
"errors": 0
},
{
"t": 21,
"ok": 18,
"errors": 0
},
{
"t": 22,
"ok": 25,
"errors": 0
},
{
"t": 23,
"ok": 26,
"errors": 0
},
{
"t": 24,
"ok": 28,
"errors": 0
},
{
"t": 25,
"ok": 28,
"errors": 0
},
{
"t": 26,
"ok": 27,
"errors": 0
},
{
"t": 27,
"ok": 17,
"errors": 0
},
{
"t": 28,
"ok": 17,
"errors": 0
},
{
"t": 29,
"ok": 15,
"errors": 0
},
{
"t": 30,
"ok": 27,
"errors": 0
},
{
"t": 31,
"ok": 28,
"errors": 0
},
{
"t": 32,
"ok": 30,
"errors": 0
},
{
"t": 33,
"ok": 29,
"errors": 0
},
{
"t": 34,
"ok": 29,
"errors": 0
},
{
"t": 35,
"ok": 15,
"errors": 0
},
{
"t": 36,
"ok": 14,
"errors": 0
},
{
"t": 37,
"ok": 25,
"errors": 0
},
{
"t": 38,
"ok": 27,
"errors": 0
},
{
"t": 39,
"ok": 25,
"errors": 0
},
{
"t": 40,
"ok": 30,
"errors": 0
},
{
"t": 41,
"ok": 29,
"errors": 0
},
{
"t": 42,
"ok": 19,
"errors": 0
},
{
"t": 43,
"ok": 16,
"errors": 0
},
{
"t": 44,
"ok": 33,
"errors": 0
},
{
"t": 45,
"ok": 50,
"errors": 0
},
{
"t": 46,
"ok": 48,
"errors": 0
},
{
"t": 47,
"ok": 49,
"errors": 0
},
{
"t": 48,
"ok": 54,
"errors": 0
},
{
"t": 49,
"ok": 48,
"errors": 0
},
{
"t": 50,
"ok": 29,
"errors": 0
},
{
"t": 51,
"ok": 36,
"errors": 0
},
{
"t": 52,
"ok": 53,
"errors": 0
},
{
"t": 53,
"ok": 59,
"errors": 0
},
{
"t": 54,
"ok": 56,
"errors": 0
},
{
"t": 55,
"ok": 20,
"errors": 0
}
],
"acked_messages": 911,
"messages_in_history": 911,
"lost_acked_messages": 0,
"duplicate_messages": 0
},
{
"stopped_replica": "945116f913cc",
"baseline_before_stop": {
"seconds": 7.66,
"successful_requests": 319,
"failed_requests": 0,
"error_reasons": {},
"error_ratio": 0.0,
"successful_rps": 41.6,
"latency_ms": {
"p50": 139.5,
"p95": 516.2,
"p99": 694.5
}
},
"stop_to_restore": {
"seconds": 19.29,
"successful_requests": 507,
"failed_requests": 0,
"error_reasons": {},
"error_ratio": 0.0,
"successful_rps": 26.3,
"latency_ms": {
"p50": 289.9,
"p95": 578.7,
"p99": 688.3
}
},
"after_restore": {
"seconds": 29.42,
"successful_requests": 962,
"failed_requests": 0,
"error_reasons": {},
"error_ratio": 0.0,
"successful_rps": 32.7,
"latency_ms": {
"p50": 239.1,
"p95": 560.3,
"p99": 699.2
}
},
"docker_stop_seconds": 4.29,
"last_error_seconds_after_stop": 0.0,
"first_success_on_other_replica_seconds": 0.06,
"requests_served_by_stopped_replica_after_stop_returned": 0,
"seconds_until_replica_back_in_rotation": 21.19,
"instances_seen": [
"3de2fdb570eb",
"945116f913cc"
],
"resource_snapshot_under_load": [
"mtusi-lab3-api-5: cpu 101.75%, mem 94.56MiB / 512MiB",
"mtusi-lab3-api-6: cpu 68.81%, mem 81.32MiB / 512MiB",
"mtusi-lab3-lb-1: cpu 15.13%, mem 17.34MiB / 128MiB",
"mtusi-lab3-postgres-1: cpu 34.54%, mem 63.73MiB / 512MiB"
],
"timeline_per_second": [
{
"t": 0,
"ok": 35,
"errors": 0
},
{
"t": 1,
"ok": 33,
"errors": 0
},
{
"t": 2,
"ok": 29,
"errors": 0
},
{
"t": 3,
"ok": 26,
"errors": 0
},
{
"t": 4,
"ok": 50,
"errors": 0
},
{
"t": 5,
"ok": 48,
"errors": 0
},
{
"t": 6,
"ok": 59,
"errors": 0
},
{
"t": 7,
"ok": 59,
"errors": 0
},
{
"t": 8,
"ok": 43,
"errors": 0
},
{
"t": 9,
"ok": 16,
"errors": 0
},
{
"t": 10,
"ok": 26,
"errors": 0
},
{
"t": 11,
"ok": 28,
"errors": 0
},
{
"t": 12,
"ok": 27,
"errors": 0
},
{
"t": 13,
"ok": 28,
"errors": 0
},
{
"t": 14,
"ok": 30,
"errors": 0
},
{
"t": 15,
"ok": 23,
"errors": 0
},
{
"t": 16,
"ok": 16,
"errors": 0
},
{
"t": 17,
"ok": 20,
"errors": 0
},
{
"t": 18,
"ok": 30,
"errors": 0
},
{
"t": 19,
"ok": 29,
"errors": 0
},
{
"t": 20,
"ok": 30,
"errors": 0
},
{
"t": 21,
"ok": 28,
"errors": 0
},
{
"t": 22,
"ok": 27,
"errors": 0
},
{
"t": 23,
"ok": 15,
"errors": 0
},
{
"t": 24,
"ok": 16,
"errors": 0
},
{
"t": 25,
"ok": 26,
"errors": 0
},
{
"t": 26,
"ok": 29,
"errors": 0
},
{
"t": 27,
"ok": 27,
"errors": 0
},
{
"t": 28,
"ok": 27,
"errors": 0
},
{
"t": 29,
"ok": 29,
"errors": 0
},
{
"t": 30,
"ok": 16,
"errors": 0
},
{
"t": 31,
"ok": 16,
"errors": 0
},
{
"t": 32,
"ok": 26,
"errors": 0
},
{
"t": 33,
"ok": 27,
"errors": 0
},
{
"t": 34,
"ok": 28,
"errors": 0
},
{
"t": 35,
"ok": 31,
"errors": 0
},
{
"t": 36,
"ok": 29,
"errors": 0
},
{
"t": 37,
"ok": 21,
"errors": 0
},
{
"t": 38,
"ok": 18,
"errors": 0
},
{
"t": 39,
"ok": 19,
"errors": 0
},
{
"t": 40,
"ok": 29,
"errors": 0
},
{
"t": 41,
"ok": 30,
"errors": 0
},
{
"t": 42,
"ok": 27,
"errors": 0
},
{
"t": 43,
"ok": 27,
"errors": 0
},
{
"t": 44,
"ok": 40,
"errors": 0
},
{
"t": 45,
"ok": 26,
"errors": 0
},
{
"t": 46,
"ok": 30,
"errors": 0
},
{
"t": 47,
"ok": 53,
"errors": 0
},
{
"t": 48,
"ok": 55,
"errors": 0
},
{
"t": 49,
"ok": 56,
"errors": 0
},
{
"t": 50,
"ok": 29,
"errors": 0
},
{
"t": 51,
"ok": 41,
"errors": 0
},
{
"t": 52,
"ok": 38,
"errors": 0
},
{
"t": 53,
"ok": 35,
"errors": 0
},
{
"t": 54,
"ok": 49,
"errors": 0
},
{
"t": 55,
"ok": 63,
"errors": 0
},
{
"t": 56,
"ok": 20,
"errors": 0
}
],
"acked_messages": 894,
"messages_in_history": 1805,
"lost_acked_messages": 0,
"duplicate_messages": 0
}
]
}