forked from mtusi2026/mtusi2026lab3
- X-Instance-Id в каждом ответе; обязательный Idempotency-Key, таблица idempotency_keys (миграция 0002, одна транзакция, конкурентные повторы через уникальный индекс) - graceful shutdown: SIGTERM -> ready=503 -> drain 3 с -> закрытие порта - compose: lb (HAProxy) + api x2 без портов + разовый migrate + postgres, лимиты CPU/RAM - хуки stop-one/restart/restore-replicas, scripts/experiments.py (failover, idempotency, rolling) - свои тесты: 44 (13 новых); evidence: smoke, pytest, persistence, failover, эксперименты
713 lines
13 KiB
JSON
713 lines
13 KiB
JSON
{
|
|
"method": "closed loop 8 workers: POST message (retry same Idempotency-Key on error) + GET page; client timeout 2s; each replica stopped with docker stop -t 8, restored, then the next",
|
|
"replicas": [
|
|
"bb53391a07f1",
|
|
"e51c99b5436b"
|
|
],
|
|
"warmup_seconds": 5,
|
|
"seconds_down": 15,
|
|
"runs": [
|
|
{
|
|
"stopped_replica": "bb53391a07f1",
|
|
"baseline_before_stop": {
|
|
"seconds": 7.73,
|
|
"successful_requests": 339,
|
|
"failed_requests": 0,
|
|
"error_reasons": {},
|
|
"error_ratio": 0.0,
|
|
"successful_rps": 43.8,
|
|
"latency_ms": {
|
|
"p50": 147.3,
|
|
"p95": 398.0,
|
|
"p99": 597.8
|
|
}
|
|
},
|
|
"stop_to_restore": {
|
|
"seconds": 19.62,
|
|
"successful_requests": 467,
|
|
"failed_requests": 0,
|
|
"error_reasons": {},
|
|
"error_ratio": 0.0,
|
|
"successful_rps": 23.8,
|
|
"latency_ms": {
|
|
"p50": 314.1,
|
|
"p95": 602.1,
|
|
"p99": 699.8
|
|
}
|
|
},
|
|
"after_restore": {
|
|
"seconds": 29.0,
|
|
"successful_requests": 894,
|
|
"failed_requests": 0,
|
|
"error_reasons": {},
|
|
"error_ratio": 0.0,
|
|
"successful_rps": 30.8,
|
|
"latency_ms": {
|
|
"p50": 253.7,
|
|
"p95": 590.4,
|
|
"p99": 704.4
|
|
}
|
|
},
|
|
"docker_stop_seconds": 4.62,
|
|
"last_error_seconds_after_stop": 0.0,
|
|
"first_success_on_other_replica_seconds": 0.02,
|
|
"requests_served_by_stopped_replica_after_stop_returned": 0,
|
|
"seconds_until_replica_back_in_rotation": 20.86,
|
|
"instances_seen": [
|
|
"bb53391a07f1",
|
|
"e51c99b5436b"
|
|
],
|
|
"resource_snapshot_under_load": [
|
|
"mtusi-lab3-api-1: cpu 90.61%, mem 81.91MiB / 512MiB",
|
|
"mtusi-lab3-api-2: cpu 103.60%, mem 82.59MiB / 512MiB",
|
|
"mtusi-lab3-lb-1: cpu 15.84%, mem 13.01MiB / 128MiB",
|
|
"mtusi-lab3-postgres-1: cpu 57.27%, mem 54.5MiB / 512MiB"
|
|
],
|
|
"timeline_per_second": [
|
|
{
|
|
"t": 0,
|
|
"ok": 34,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 1,
|
|
"ok": 45,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 2,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 3,
|
|
"ok": 45,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 4,
|
|
"ok": 50,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 5,
|
|
"ok": 48,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 6,
|
|
"ok": 54,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 7,
|
|
"ok": 48,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 8,
|
|
"ok": 32,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 9,
|
|
"ok": 16,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 10,
|
|
"ok": 23,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 11,
|
|
"ok": 22,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 12,
|
|
"ok": 21,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 13,
|
|
"ok": 26,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 14,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 15,
|
|
"ok": 18,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 16,
|
|
"ok": 15,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 17,
|
|
"ok": 18,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 18,
|
|
"ok": 26,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 19,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 20,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 21,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 22,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 23,
|
|
"ok": 16,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 24,
|
|
"ok": 13,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 25,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 26,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 27,
|
|
"ok": 25,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 28,
|
|
"ok": 26,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 29,
|
|
"ok": 25,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 30,
|
|
"ok": 18,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 31,
|
|
"ok": 15,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 32,
|
|
"ok": 22,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 33,
|
|
"ok": 24,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 34,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 35,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 36,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 37,
|
|
"ok": 19,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 38,
|
|
"ok": 16,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 39,
|
|
"ok": 16,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 40,
|
|
"ok": 25,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 41,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 42,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 43,
|
|
"ok": 25,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 44,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 45,
|
|
"ok": 38,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 46,
|
|
"ok": 22,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 47,
|
|
"ok": 40,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 48,
|
|
"ok": 51,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 49,
|
|
"ok": 56,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 50,
|
|
"ok": 38,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 51,
|
|
"ok": 37,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 52,
|
|
"ok": 49,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 53,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 54,
|
|
"ok": 40,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 55,
|
|
"ok": 56,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 56,
|
|
"ok": 22,
|
|
"errors": 0
|
|
}
|
|
],
|
|
"acked_messages": 850,
|
|
"messages_in_history": 850,
|
|
"lost_acked_messages": 0,
|
|
"duplicate_messages": 0
|
|
},
|
|
{
|
|
"stopped_replica": "e51c99b5436b",
|
|
"baseline_before_stop": {
|
|
"seconds": 7.62,
|
|
"successful_requests": 295,
|
|
"failed_requests": 0,
|
|
"error_reasons": {},
|
|
"error_ratio": 0.0,
|
|
"successful_rps": 38.7,
|
|
"latency_ms": {
|
|
"p50": 169.1,
|
|
"p95": 520.0,
|
|
"p99": 644.0
|
|
}
|
|
},
|
|
"stop_to_restore": {
|
|
"seconds": 19.96,
|
|
"successful_requests": 465,
|
|
"failed_requests": 0,
|
|
"error_reasons": {},
|
|
"error_ratio": 0.0,
|
|
"successful_rps": 23.3,
|
|
"latency_ms": {
|
|
"p50": 321.7,
|
|
"p95": 602.9,
|
|
"p99": 694.0
|
|
}
|
|
},
|
|
"after_restore": {
|
|
"seconds": 29.98,
|
|
"successful_requests": 966,
|
|
"failed_requests": 0,
|
|
"error_reasons": {},
|
|
"error_ratio": 0.0,
|
|
"successful_rps": 32.2,
|
|
"latency_ms": {
|
|
"p50": 237.7,
|
|
"p95": 579.4,
|
|
"p99": 767.1
|
|
}
|
|
},
|
|
"docker_stop_seconds": 4.96,
|
|
"last_error_seconds_after_stop": 0.0,
|
|
"first_success_on_other_replica_seconds": 0.0,
|
|
"requests_served_by_stopped_replica_after_stop_returned": 0,
|
|
"seconds_until_replica_back_in_rotation": 21.78,
|
|
"instances_seen": [
|
|
"bb53391a07f1",
|
|
"e51c99b5436b"
|
|
],
|
|
"resource_snapshot_under_load": [
|
|
"mtusi-lab3-api-1: cpu 103.71%, mem 82.34MiB / 512MiB",
|
|
"mtusi-lab3-api-2: cpu 89.86%, mem 90.22MiB / 512MiB",
|
|
"mtusi-lab3-lb-1: cpu 24.15%, mem 13.37MiB / 128MiB",
|
|
"mtusi-lab3-postgres-1: cpu 34.28%, mem 54.15MiB / 512MiB"
|
|
],
|
|
"timeline_per_second": [
|
|
{
|
|
"t": 0,
|
|
"ok": 19,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 1,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 2,
|
|
"ok": 53,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 3,
|
|
"ok": 35,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 4,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 5,
|
|
"ok": 53,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 6,
|
|
"ok": 52,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 7,
|
|
"ok": 38,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 8,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 9,
|
|
"ok": 24,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 10,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 11,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 12,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 13,
|
|
"ok": 22,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 14,
|
|
"ok": 19,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 15,
|
|
"ok": 15,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 16,
|
|
"ok": 17,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 17,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 18,
|
|
"ok": 21,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 19,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 20,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 21,
|
|
"ok": 24,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 22,
|
|
"ok": 13,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 23,
|
|
"ok": 13,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 24,
|
|
"ok": 21,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 25,
|
|
"ok": 26,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 26,
|
|
"ok": 30,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 27,
|
|
"ok": 27,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 28,
|
|
"ok": 24,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 29,
|
|
"ok": 20,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 30,
|
|
"ok": 15,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 31,
|
|
"ok": 16,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 32,
|
|
"ok": 22,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 33,
|
|
"ok": 24,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 34,
|
|
"ok": 25,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 35,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 36,
|
|
"ok": 21,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 37,
|
|
"ok": 13,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 38,
|
|
"ok": 13,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 39,
|
|
"ok": 16,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 40,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 41,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 42,
|
|
"ok": 24,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 43,
|
|
"ok": 28,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 44,
|
|
"ok": 29,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 45,
|
|
"ok": 25,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 46,
|
|
"ok": 39,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 47,
|
|
"ok": 49,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 48,
|
|
"ok": 51,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 49,
|
|
"ok": 57,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 50,
|
|
"ok": 50,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 51,
|
|
"ok": 34,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 52,
|
|
"ok": 31,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 53,
|
|
"ok": 47,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 54,
|
|
"ok": 57,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 55,
|
|
"ok": 55,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 56,
|
|
"ok": 54,
|
|
"errors": 0
|
|
},
|
|
{
|
|
"t": 57,
|
|
"ok": 31,
|
|
"errors": 0
|
|
}
|
|
],
|
|
"acked_messages": 863,
|
|
"messages_in_history": 1713,
|
|
"lost_acked_messages": 0,
|
|
"duplicate_messages": 0
|
|
}
|
|
]
|
|
}
|