Record successful clean-clone fault and persistence verification

This commit is contained in:
2026-10-16 16:13:31 +03:00
parent dec47a0b40
commit 504188c920
3 changed files with 551 additions and 0 deletions
+13
View File
@@ -108,6 +108,19 @@ HAProxy round-robin использует Docker DNS `127.0.0.11` и 4 динам
Raw и агрегаты: `evidence/scaling/db-intervention.json`, `db-summary.json`, CSV и `*-resources.json`. В первой попытке смена бюджета через пересоздание БД привела к преждевременному завершению `compose --wait` на ещё unhealthy API. API затем восстановились сами. Незаконченный запуск сохранён в `interrupted-db-intervention.json`; в окончательные парные показатели он не входит. Сценарий исправлен на изменение квоты без рестарта. Первая законченная серия 1/2/4 сохраняет отдельные перезапуски перед замером; между сериями есть разогрев/шум, поэтому сравнение улучшения опирается именно на чередующиеся пары.
## Проверка чистой копии
16.10.2026 локальный `git clone --local --branch solution/bvt2301-paramonov`
в отдельный каталог поднят с новыми secrets и новым PostgreSQL volume.
Build, миграции и полный `scripts/verify_stack.py` прошли. 32 параллельных повтора
дали одно сообщение, public smoke/persistence/failover прошли. При остановке каждой
реплики сохранены 133/133 и 241/241 подтверждения; при rolling replacement —
1184/1184. Raw итог: `evidence/clean-clone-integration.json`. В сценарии восстановления
добавлено ограниченное ожидание фактического healthy всех реплик: Compose 2.33 может
преждевременно вернуть старый unhealthy сразу после повторного запуска остановленного
контейнера. Проверка не игнорирует этот статус, а ждёт его настоящего восстановления.
Этот прогон проверяет воспроизводимость и сохранность, не входит в сравнение RPS.
## Ограничения и источники
Windows 10, Intel i5-11400H, 8 GB RAM; WSL2 ограничен 4 CPU / 3 GB RAM / 4 GB swap. Docker 28.0.1, Compose 2.33.1. Генератор и Docker находятся на одном физическом компьютере. Короткий closed-loop эксперимент имеет coordinated omission и заметный шум планировщика; это наблюдения локального стенда, не production SLA и не верхняя граница мощности сервиса. На высокой параллельности генератор сам может упираться в одно ядро.
+83
View File
@@ -0,0 +1,83 @@
{
"public_smoke": "PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.",
"parallel_idempotency": {
"attempts": 32,
"replicas": 2,
"messages": 1
},
"key_scopes_validation": "409 mismatch; independent users/chats; missing/invalid key 400; 400 does not reserve key",
"public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.",
"replay_after_restart_logout": "same response after all-service restart; revoked session cannot replay",
"public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 89, \"failed_attempts\": 3, \"first_recovery_seconds\": 9.547}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.",
"each_replica_failure": [
{
"events": [
{
"container": "c9fbf8dae42c7578f4a67caa5a3925e73bf80e74d96e737ac71fa6b41b3ca409",
"alias": "9e8bbf3e8e2941e9887710df65a24df5",
"signal_at": 4.360000000000582,
"stopped_at": 7.390999999995984,
"GET_recovery_seconds": 3.031,
"POST_recovery_seconds": 3.093
}
],
"duration_seconds": 14.063,
"attempts": 268,
"successful_rps": 18.915,
"errors": 2,
"confirmed_messages": 133,
"stored_messages": 133,
"all_confirmed_retained": true
},
{
"events": [
{
"container": "eea6321979527b7d0b71d2d61e0b5c78c9f9bad71f89af3bcc267e18c5cf6984",
"alias": "579aca79bdc94c24ab34cc4f2e0a1856",
"signal_at": 7.094000000004598,
"stopped_at": 11.406000000002678,
"GET_recovery_seconds": 4.765,
"POST_recovery_seconds": 4.375
}
],
"duration_seconds": 17.641,
"attempts": 572,
"successful_rps": 29.363,
"errors": 54,
"confirmed_messages": 241,
"stored_messages": 241,
"all_confirmed_retained": true
}
],
"rolling": {
"events": [
{
"container": "c9fbf8dae42c7578f4a67caa5a3925e73bf80e74d96e737ac71fa6b41b3ca409",
"alias": "7d67144e17b849cca5a7fe9acfa98e67",
"signal_at": 7.811999999998079,
"stopped_at": 10.328000000001339,
"replacement_ready_at": 38.20300000000134,
"GET_recovery_seconds": 2.531,
"POST_recovery_seconds": 2.531
},
{
"container": "eea6321979527b7d0b71d2d61e0b5c78c9f9bad71f89af3bcc267e18c5cf6984",
"alias": "2f061fd705ef44ae98ec7625d7553eb3",
"signal_at": 44.20300000000134,
"stopped_at": 46.38999999999942,
"replacement_ready_at": 55.45300000000134,
"GET_recovery_seconds": 2.297,
"POST_recovery_seconds": 2.437
}
],
"duration_seconds": 62.859,
"attempts": 2438,
"successful_rps": 37.783,
"errors": 63,
"confirmed_messages": 1184,
"stored_messages": 1184,
"all_confirmed_retained": true
},
"migration_repeat": "Already applied: 001_initial.sql\nAlready applied: 002_idempotency.sql",
"passed": true
}
+455
View File
@@ -0,0 +1,455 @@
{
"before": {
"c9fbf8dae42c7578f4a67caa5a3925e73bf80e74d96e737ac71fa6b41b3ca409": {
"service": "api",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 65282,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T13:07:33.385611585Z",
"FinishedAt": "2026-10-16T13:07:20.959753988Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T13:08:17.757617915Z",
"End": "2026-10-16T13:08:18.461928144Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:23.463326789Z",
"End": "2026-10-16T13:08:24.343124107Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:29.344340959Z",
"End": "2026-10-16T13:08:30.182047181Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:35.426083348Z",
"End": "2026-10-16T13:08:36.328986972Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:41.330146647Z",
"End": "2026-10-16T13:08:42.695818974Z",
"ExitCode": 0,
"Output": ""
}
]
}
}
},
"eea6321979527b7d0b71d2d61e0b5c78c9f9bad71f89af3bcc267e18c5cf6984": {
"service": "api",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 63618,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T13:06:39.005903832Z",
"FinishedAt": "2026-10-16T13:06:38.189103867Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T13:08:16.188254944Z",
"End": "2026-10-16T13:08:17.072622725Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:22.074430958Z",
"End": "2026-10-16T13:08:23.043746212Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:28.045095989Z",
"End": "2026-10-16T13:08:28.945020302Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:34.189062899Z",
"End": "2026-10-16T13:08:35.304272091Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:40.305120885Z",
"End": "2026-10-16T13:08:41.993436948Z",
"ExitCode": 0,
"Output": ""
}
]
}
}
},
"52299d4b6941c54401ddc9634a44ba8dbfb81eebe0734dc9246f0746dc98d2a0": {
"service": "db",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 63483,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T13:06:38.049287146Z",
"FinishedAt": "2026-10-16T13:06:37.441191923Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T13:08:37.058806656Z",
"End": "2026-10-16T13:08:37.145713206Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T13:08:39.147345378Z",
"End": "2026-10-16T13:08:39.290057744Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T13:08:41.292168542Z",
"End": "2026-10-16T13:08:41.403378494Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T13:08:43.404471241Z",
"End": "2026-10-16T13:08:43.494522228Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T13:08:45.495326434Z",
"End": "2026-10-16T13:08:45.5599225Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
}
]
}
}
},
"43c22160d55b9ed46ad3891d519002e543fc496d998f45fe3a141d3ce8197f44": {
"service": "cache",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 63429,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T13:06:37.976782113Z",
"FinishedAt": "2026-10-16T13:06:37.378727846Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T13:08:37.868863667Z",
"End": "2026-10-16T13:08:37.915742781Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T13:08:39.91699778Z",
"End": "2026-10-16T13:08:39.961015681Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T13:08:41.962124483Z",
"End": "2026-10-16T13:08:42.006901301Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T13:08:44.007677982Z",
"End": "2026-10-16T13:08:44.0520358Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T13:08:46.053045151Z",
"End": "2026-10-16T13:08:46.091323762Z",
"ExitCode": 0,
"Output": "PONG\n"
}
]
}
}
},
"3bc2665b06cc293a95628179c685386923d242c93c61b901101d65b2956d07b8": {
"service": "lb",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 63578,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T13:06:38.318501373Z",
"FinishedAt": "2026-10-16T13:06:37.340765782Z"
}
}
},
"after": {
"c9fbf8dae42c7578f4a67caa5a3925e73bf80e74d96e737ac71fa6b41b3ca409": {
"service": "api",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 67546,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T13:08:53.144468207Z",
"FinishedAt": "2026-10-16T13:08:52.555817749Z",
"Health": {
"Status": "starting",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T13:08:23.463326789Z",
"End": "2026-10-16T13:08:24.343124107Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:29.344340959Z",
"End": "2026-10-16T13:08:30.182047181Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:35.426083348Z",
"End": "2026-10-16T13:08:36.328986972Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:41.330146647Z",
"End": "2026-10-16T13:08:42.695818974Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:47.69778778Z",
"End": "2026-10-16T13:08:48.80249648Z",
"ExitCode": 0,
"Output": ""
}
]
}
}
},
"eea6321979527b7d0b71d2d61e0b5c78c9f9bad71f89af3bcc267e18c5cf6984": {
"service": "api",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 67623,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T13:08:53.87654863Z",
"FinishedAt": "2026-10-16T13:08:53.333764617Z",
"Health": {
"Status": "starting",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T13:08:22.074430958Z",
"End": "2026-10-16T13:08:23.043746212Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:28.045095989Z",
"End": "2026-10-16T13:08:28.945020302Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:34.189062899Z",
"End": "2026-10-16T13:08:35.304272091Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:40.305120885Z",
"End": "2026-10-16T13:08:41.993436948Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2026-10-16T13:08:46.994665863Z",
"End": "2026-10-16T13:08:48.196044375Z",
"ExitCode": 0,
"Output": ""
}
]
}
}
},
"52299d4b6941c54401ddc9634a44ba8dbfb81eebe0734dc9246f0746dc98d2a0": {
"service": "db",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 67309,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T13:08:50.279865768Z",
"FinishedAt": "2026-10-16T13:08:49.350953208Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T13:08:43.404471241Z",
"End": "2026-10-16T13:08:43.494522228Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T13:08:45.495326434Z",
"End": "2026-10-16T13:08:45.5599225Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T13:08:47.561452382Z",
"End": "2026-10-16T13:08:47.675438588Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T13:08:52.796676713Z",
"End": "2026-10-16T13:08:53.238291485Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
},
{
"Start": "2026-10-16T13:08:55.241718062Z",
"End": "2026-10-16T13:08:55.561496948Z",
"ExitCode": 0,
"Output": "/var/run/postgresql:5432 - accepting connections\n"
}
]
}
}
},
"43c22160d55b9ed46ad3891d519002e543fc496d998f45fe3a141d3ce8197f44": {
"service": "cache",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 67286,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T13:08:50.148136164Z",
"FinishedAt": "2026-10-16T13:08:49.213263652Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2026-10-16T13:08:44.007677982Z",
"End": "2026-10-16T13:08:44.0520358Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T13:08:46.053045151Z",
"End": "2026-10-16T13:08:46.091323762Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T13:08:48.092247452Z",
"End": "2026-10-16T13:08:48.168296007Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T13:08:52.768656484Z",
"End": "2026-10-16T13:08:52.911218174Z",
"ExitCode": 0,
"Output": "PONG\n"
},
{
"Start": "2026-10-16T13:08:54.913086539Z",
"End": "2026-10-16T13:08:55.056396718Z",
"ExitCode": 0,
"Output": "PONG\n"
}
]
}
}
},
"3bc2665b06cc293a95628179c685386923d242c93c61b901101d65b2956d07b8": {
"service": "lb",
"state": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 67366,
"ExitCode": 0,
"Error": "",
"StartedAt": "2026-10-16T13:08:50.555227454Z",
"FinishedAt": "2026-10-16T13:08:49.428410831Z"
}
}
},
"volumes_removed": false
}