- X-Instance-Id в каждом ответе; обязательный Idempotency-Key, таблица idempotency_keys (миграция 0002, одна транзакция, конкурентные повторы через уникальный индекс) - graceful shutdown: SIGTERM -> ready=503 -> drain 3 с -> закрытие порта - compose: lb (HAProxy) + api x2 без портов + разовый migrate + postgres, лимиты CPU/RAM - хуки stop-one/restart/restore-replicas, scripts/experiments.py (failover, idempotency, rolling) - свои тесты: 44 (13 новых); evidence: smoke, pytest, persistence, failover, эксперименты
703 lines
13 KiB
JSON
703 lines
13 KiB
JSON
{
|
|
"method": "closed loop 8 workers: POST message (retry same Idempotency-Key on error) + GET page; client timeout 2s; each replica stopped with docker stop -t 8, restored, then the next",
|
|
"replicas": [
|
|
"3de2fdb570eb",
|
|
"945116f913cc"
|
|
],
|
|
"warmup_seconds": 5,
|
|
"seconds_down": 15,
|
|
"runs": [
|
|
{
|
|
"stopped_replica": "3de2fdb570eb",
|
|
"baseline_before_stop": {
|
|
"seconds": 7.72,
|
|
"successful_requests": 359,
|
|
"failed_requests": 0,
|
|
"error_reasons": {},
|
|
"error_ratio": 0.0,
|
|
"successful_rps": 46.5,
|
|
"latency_ms": {
|
|
"p50": 136.4,
|
|
"p95": 433.5,
|
|
"p99": 628.8
|
|
}
|
|
},
|
|
"stop_to_restore": {
|
|
"seconds": 19.38,
|
|
"successful_requests": 538,
|
|
"failed_requests": 0,
|
|
"error_reasons": {},
|
|
"error_ratio": 0.0,
|
|
"successful_rps": 27.8,
|
|
"latency_ms": {
|
|
"p50": 281.4,
|
|
"p95": 515.9,
|
|
"p99": 694.2
|
|
}
|
|
},
|
|
"after_restore": {
|
|
"seconds": 28.26,
|
|
"successful_requests": 925,
|
|
"failed_requests": 0,
|
|
"error_reasons": {},
|
|
"error_ratio": 0.0,
|
|
"successful_rps": 32.7,
|
|
"latency_ms": {
|
|
"p50": 244.6,
|
|
"p95": 537.0,
|
|
"p99": 698.6
|
|
}
|
|
},
|
|
"docker_stop_seconds": 4.38,
|
|
"last_error_seconds_after_stop": 0.0,
|
|
"first_success_on_other_replica_seconds": 0.03,
|
|
"requests_served_by_stopped_replica_after_stop_returned": 0,
|
|
"seconds_until_replica_back_in_rotation": 20.11,
|
|
"instances_seen": [
|
|
"3de2fdb570eb",
|
|
"945116f913cc"
|
|
],
|
|
"resource_snapshot_under_load": [
|
|
"mtusi-lab3-api-5: cpu 99.82%, mem 80.75MiB / 512MiB",
|
|
"mtusi-lab3-api-6: cpu 46.77%, mem 81.65MiB / 512MiB",
|
|
"mtusi-lab3-lb-1: cpu 7.09%, mem 17.03MiB / 128MiB",
|
|
"mtusi-lab3-postgres-1: cpu 25.45%, mem 64.93MiB / 512MiB"
|
|
],
|
|
"timeline_per_second": [
|
|
{
|
|
"t": 0,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 1,
|
|
"ok": 58,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 2,
|
|
"ok": 60,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 3,
|
|
"ok": 59,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 4,
|
|
"ok": 60,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 5,
|
|
"ok": 35,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 6,
|
|
"ok": 33,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 7,
|
|
"ok": 41,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 8,
|
|
"ok": 56,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 9,
|
|
"ok": 38,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 10,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 11,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 12,
|
|
"ok": 23,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 13,
|
|
"ok": 17,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 14,
|
|
"ok": 15,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 15,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 16,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 17,
|
|
"ok": 31,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 18,
|
|
"ok": 26,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 19,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 20,
|
|
"ok": 17,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 21,
|
|
"ok": 18,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 22,
|
|
"ok": 25,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 23,
|
|
"ok": 26,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 24,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 25,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 26,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 27,
|
|
"ok": 17,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 28,
|
|
"ok": 17,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 29,
|
|
"ok": 15,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 30,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 31,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 32,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 33,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 34,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 35,
|
|
"ok": 15,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 36,
|
|
"ok": 14,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 37,
|
|
"ok": 25,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 38,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 39,
|
|
"ok": 25,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 40,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 41,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 42,
|
|
"ok": 19,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 43,
|
|
"ok": 16,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 44,
|
|
"ok": 33,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 45,
|
|
"ok": 50,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 46,
|
|
"ok": 48,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 47,
|
|
"ok": 49,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 48,
|
|
"ok": 54,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 49,
|
|
"ok": 48,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 50,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 51,
|
|
"ok": 36,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 52,
|
|
"ok": 53,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 53,
|
|
"ok": 59,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 54,
|
|
"ok": 56,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 55,
|
|
"ok": 20,
|
|
"errors": 0
|
|
}
|
|
],
|
|
"acked_messages": 911,
|
|
"messages_in_history": 911,
|
|
"lost_acked_messages": 0,
|
|
"duplicate_messages": 0
|
|
},
|
|
{
|
|
"stopped_replica": "945116f913cc",
|
|
"baseline_before_stop": {
|
|
"seconds": 7.66,
|
|
"successful_requests": 319,
|
|
"failed_requests": 0,
|
|
"error_reasons": {},
|
|
"error_ratio": 0.0,
|
|
"successful_rps": 41.6,
|
|
"latency_ms": {
|
|
"p50": 139.5,
|
|
"p95": 516.2,
|
|
"p99": 694.5
|
|
}
|
|
},
|
|
"stop_to_restore": {
|
|
"seconds": 19.29,
|
|
"successful_requests": 507,
|
|
"failed_requests": 0,
|
|
"error_reasons": {},
|
|
"error_ratio": 0.0,
|
|
"successful_rps": 26.3,
|
|
"latency_ms": {
|
|
"p50": 289.9,
|
|
"p95": 578.7,
|
|
"p99": 688.3
|
|
}
|
|
},
|
|
"after_restore": {
|
|
"seconds": 29.42,
|
|
"successful_requests": 962,
|
|
"failed_requests": 0,
|
|
"error_reasons": {},
|
|
"error_ratio": 0.0,
|
|
"successful_rps": 32.7,
|
|
"latency_ms": {
|
|
"p50": 239.1,
|
|
"p95": 560.3,
|
|
"p99": 699.2
|
|
}
|
|
},
|
|
"docker_stop_seconds": 4.29,
|
|
"last_error_seconds_after_stop": 0.0,
|
|
"first_success_on_other_replica_seconds": 0.06,
|
|
"requests_served_by_stopped_replica_after_stop_returned": 0,
|
|
"seconds_until_replica_back_in_rotation": 21.19,
|
|
"instances_seen": [
|
|
"3de2fdb570eb",
|
|
"945116f913cc"
|
|
],
|
|
"resource_snapshot_under_load": [
|
|
"mtusi-lab3-api-5: cpu 101.75%, mem 94.56MiB / 512MiB",
|
|
"mtusi-lab3-api-6: cpu 68.81%, mem 81.32MiB / 512MiB",
|
|
"mtusi-lab3-lb-1: cpu 15.13%, mem 17.34MiB / 128MiB",
|
|
"mtusi-lab3-postgres-1: cpu 34.54%, mem 63.73MiB / 512MiB"
|
|
],
|
|
"timeline_per_second": [
|
|
{
|
|
"t": 0,
|
|
"ok": 35,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 1,
|
|
"ok": 33,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 2,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 3,
|
|
"ok": 26,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 4,
|
|
"ok": 50,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 5,
|
|
"ok": 48,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 6,
|
|
"ok": 59,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 7,
|
|
"ok": 59,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 8,
|
|
"ok": 43,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 9,
|
|
"ok": 16,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 10,
|
|
"ok": 26,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 11,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 12,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 13,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 14,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 15,
|
|
"ok": 23,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 16,
|
|
"ok": 16,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 17,
|
|
"ok": 20,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 18,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 19,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 20,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 21,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 22,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 23,
|
|
"ok": 15,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 24,
|
|
"ok": 16,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 25,
|
|
"ok": 26,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 26,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 27,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 28,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 29,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 30,
|
|
"ok": 16,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 31,
|
|
"ok": 16,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 32,
|
|
"ok": 26,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 33,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 34,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 35,
|
|
"ok": 31,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 36,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 37,
|
|
"ok": 21,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 38,
|
|
"ok": 18,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 39,
|
|
"ok": 19,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 40,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 41,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 42,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 43,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 44,
|
|
"ok": 40,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 45,
|
|
"ok": 26,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 46,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 47,
|
|
"ok": 53,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 48,
|
|
"ok": 55,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 49,
|
|
"ok": 56,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 50,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 51,
|
|
"ok": 41,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 52,
|
|
"ok": 38,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 53,
|
|
"ok": 35,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 54,
|
|
"ok": 49,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 55,
|
|
"ok": 63,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 56,
|
|
"ok": 20,
|
|
"errors": 0
|
|
}
|
|
],
|
|
"acked_messages": 894,
|
|
"messages_in_history": 1805,
|
|
"lost_acked_messages": 0,
|
|
"duplicate_messages": 0
|
|
}
|
|
]
|
|
}
|