From 7f848bbc421bfb23b2f6c6a9b84993c6a9966e51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=BD=D0=B5=D0=BA?= <88394552+Lunatuk@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:46:06 +0300 Subject: [PATCH] =?UTF-8?q?=D0=B8=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BE=D1=88=D0=B8=D0=B1=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 27 +- evidence/lab3-checks.json | 933 ++++++++++++++-------------- evidence/lab3-checks.txt | 432 ++++++------- evidence/postgres-pause-before.json | 10 + src/db.py | 16 + student-tests/docker_lifecycle.py | 3 + student-tests/postgres_pause.py | 81 +++ 7 files changed, 823 insertions(+), 679 deletions(-) create mode 100644 evidence/postgres-pause-before.json create mode 100644 student-tests/postgres_pause.py diff --git a/REPORT.md b/REPORT.md index 3eaa892..53f416c 100644 --- a/REPORT.md +++ b/REPORT.md @@ -86,6 +86,23 @@ HAProxy проверяет `/health/ready` каждую 1 секунду. Два Остальные таймауты HAProxy: server 8s, client 10s, чтение HTTP-заголовков 5s. `retries 1` относится к установлению backend-соединения; повторов уже отправленных HTTP POST по статусам не включено. Клиент повторяет неопределённый POST сообщения с исходным ключом. +Ожидание сетевого ответа PostgreSQL ограничено на клиенте 2 секундами: +`TimeoutConnection.wait` в `src/db.py` используется всеми соединениями пула, +включая проверку перед выдачей, SQL и завершение транзакций. При OperationalError +соединение сразу закрывается и исключается из повторного использования, чтобы +не зависнуть ещё и на rollback/cancel. Приложение возвращает контрактный +`503 unavailable` с X-Request-Id и X-Instance-Id раньше таймаута HAProxy. +Серверный statement_timeout=3s сохранён, но сам по себе он не помогает при +`docker compose pause postgres`: остановленный процесс БД не исполняет таймер. + +Регрессия `student-tests/postgres_pause.py` входит в `./scripts/check-docker.sh`. +До исправления прогретый запрос истории после паузы получил 504 за 8.008s +([исходное наблюдение](evidence/postgres-pause-before.json)). После исправления — +JSON 503 за 2.008s. Отдельно приостановлен PostgreSQL во время уже начавшегося +`pg_sleep`: клиент завершил SQL за 2.002s и закрыл соединение. После unpause +подтверждены прежняя сессия, сохранность сообщения и новая запись без рестарта API. +Результат сохранён в `postgres_pause` файла `evidence/lab3-checks.json`. + API является PID 1. При SIGTERM Uvicorn прекращает приём новых соединений, даёт текущим запросам до 5 секунд и закрывает пул. Docker даёт 8 секунд до принудительного завершения. Проверка с незавершённым телом POST измеряет отказ новых соединений и выход до 8 секунд; проверяется `Application shutdown complete`, exit code 0 и отсутствие OOM. Переходные ошибки до исключения backend из LB учитываются в нагрузочном сценарии. ## Общая идемпотентность @@ -147,17 +164,17 @@ docker compose start api ## Измерения -Дата прогона — 29.10.2026. Среда: macOS arm64, Docker VM Linux aarch64, 10 CPU, 8217448448 байт RAM. Лимиты контейнеров приведены выше. Профиль — четыре закрытых клиента: POST сообщения и GET истории с существующими сессиями, пауза 50ms между парами запросов. Для неопределённого POST сохраняются прежние ключ и тело. При отказе нагрузка длится 15 секунд после начала воздействия; при замене — весь action и минимум 3 секунды после. Перед воздействием есть 1 секунда прогрева, которая входит в общие счётчики. +Дата повторного прогона после исправления таймаута БД — 06.11.2026. Среда: macOS arm64, Docker VM Linux aarch64, 10 CPU, 8217448448 байт RAM. Лимиты контейнеров приведены выше. Профиль — четыре закрытых клиента: POST сообщения и GET истории с существующими сессиями, пауза 50ms между парами запросов. Для неопределённого POST сохраняются прежние ключ и тело. При отказе нагрузка длится 15 секунд после начала воздействия; при замене — весь action и минимум 3 секунды после. Перед воздействием есть 1 секунда прогрева, которая входит в общие счётчики. Численные результаты последнего успешного прогона приведены ниже. | Воздействие | Длительность, с | Успешные HTTP/с | Ошибки | Макс. пауза POST / GET, с | Потеря подтверждённых / дубли | | --- | ---: | ---: | ---: | --- | --- | -| Остановка 7f6a80b69e32 | 16.009 | 98.44 | 4 | 1.957 / 2.004 | 0 / 0 | -| Остановка f973eaec4d52 | 16.015 | 98.28 | 4 | 2.006 / 2.006 | 0 / 0 | -| Поочерёдная замена обоих API | 23.774 | 89.64 | 11 | 1.059 / 2.034 | 0 / 0 | +| Остановка 8e14050120c6 | 16.014 | 90.67 | 4 | 2.009 / 2.003 | 0 / 0 | +| Остановка 998bd7b19b89 | 16.015 | 93.91 | 4 | 1.968 / 1.972 | 0 / 0 | +| Поочерёдная замена обоих API | 22.757 | 91.05 | 8 | 1.007 / 1.074 | 0 / 0 | -SIGTERM с незавершённым POST: новые соединения перестали приниматься за 0.167 с, выход за 5.393 с, exit code 0, OOM отсутствует. При остановке Valkey и PostgreSQL по отдельности подтверждены API live=200, API ready=503 и LB ready=503; после восстановления прежний токен продолжил работать. +SIGTERM с незавершённым POST: новые соединения перестали приниматься за 0.161 с, выход за 5.454 с, exit code 0, OOM отсутствует. При остановке Valkey и PostgreSQL по отдельности подтверждены API live=200, API ready=503 и LB ready=503; после восстановления прежний токен продолжил работать. RPS считается по всем успешным HTTP-запросам чтения и записи за полную длительность сценария; ошибки считаются отдельно. «Первый успех» показывает продолжение обслуживания после начала action, а максимальный промежуток между успешными запросами дополнительно показывает паузы во всём сценарии. После каждого воздействия проверяются все подтверждённые сообщения, исходное сообщение до отказа, отсутствие повторяющихся ID/текстов и работа прежней сессии. diff --git a/evidence/lab3-checks.json b/evidence/lab3-checks.json index 8c49713..8028f12 100644 --- a/evidence/lab3-checks.json +++ b/evidence/lab3-checks.json @@ -1,7 +1,7 @@ { - "utc": "2026-10-29T10:13:09.874202+00:00", - "project": "lab3-check-a31f544685", - "base_url": "http://127.0.0.1:58474", + "utc": "2026-11-06T07:34:22.361174+00:00", + "project": "lab3-check-2418557caa", + "base_url": "http://127.0.0.1:64150", "host": "Darwin arm64", "docker": "29.0.1 aarch64 CPUs=10 RAM=8217448448", "profile": { @@ -10,39 +10,15 @@ "operation": "POST message then GET history, same-key retry after uncertain POST" }, "compose_version": "2.40.3-desktop.1", - "source_commit": "74aa2242515a05192eceb4e3838e3eb4f032f397", + "source_commit": "64ea89d04046a0d4eb761c8774d0b33cdb363bb0", "working_tree": true, "containers": [ { - "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", - "instance": "7f6a80b69e32", + "id": "8e14050120c607ce540e41301256a08292ff7b832d7b32682871ad3c1facf560", + "instance": "8e14050120c6", "service": "api", - "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", - "started_at": "2026-10-29T10:13:15.412188555Z", - "running": true, - "exit_code": 0, - "memory_bytes": 268435456, - "nano_cpus": 1000000000, - "published_ports": {}, - "mounts": [ - { - "type": "bind", - "destination": "/run/secrets/cache_url", - "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/database_url", - "name": null - } - ] - }, - { - "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", - "instance": "f973eaec4d52", - "service": "api", - "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", - "started_at": "2026-10-29T10:13:15.325770597Z", + "image": "sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f", + "started_at": "2026-11-06T07:34:27.834668252Z", "running": true, "exit_code": 0, "memory_bytes": 268435456, @@ -62,11 +38,35 @@ ] }, { - "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", - "instance": "2abbb539f04c", + "id": "998bd7b19b8910cbf3e823a15144b6bdbe42a64e15bb9ae602cdbc30a7e3e190", + "instance": "998bd7b19b89", + "service": "api", + "image": "sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f", + "started_at": "2026-11-06T07:34:27.940669294Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "28c0b15ec1e6fa82db624d0742fdeae73aaff4b60bac2ea7ab49a02ff0922f6c", + "instance": "28c0b15ec1e6", "service": "lb", "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", - "started_at": "2026-10-29T10:13:21.014548544Z", + "started_at": "2026-11-06T07:34:33.562351255Z", "running": true, "exit_code": 0, "memory_bytes": 67108864, @@ -75,7 +75,7 @@ "8080/tcp": [ { "HostIp": "127.0.0.1", - "HostPort": "58474" + "HostPort": "64150" } ] }, @@ -88,11 +88,11 @@ ] }, { - "id": "a0a498f6df2fa9b5b03b24cb1ef2f3786139a8f744f71a6803e414867c391f94", - "instance": "a0a498f6df2f", + "id": "b6926643ca9f5b8537a02918d004adcaad6353a61f53bc821311e251f15299a1", + "instance": "b6926643ca9f", "service": "migrate", - "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", - "started_at": "2026-10-29T10:13:14.212270221Z", + "image": "sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f", + "started_at": "2026-11-06T07:34:26.723215668Z", "running": false, "exit_code": 0, "memory_bytes": 268435456, @@ -107,11 +107,11 @@ ] }, { - "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", - "instance": "653f44c2806d", + "id": "c4dc7a588242d47cc065defca1612a0b818eae805c3643de642f0672ff6e2de4", + "instance": "c4dc7a588242", "service": "postgres", "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", - "started_at": "2026-10-29T10:13:11.598535928Z", + "started_at": "2026-11-06T07:34:24.113683417Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -121,7 +121,7 @@ { "type": "volume", "destination": "/var/lib/postgresql/data", - "name": "lab3-check-a31f544685_postgres_data" + "name": "lab3-check-2418557caa_postgres_data" }, { "type": "bind", @@ -131,11 +131,11 @@ ] }, { - "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", - "instance": "4aca2308d7a0", + "id": "5e34f9349f45e6b95cb35a0f6cf50bce63c8158369c91715140436f2394aaa09", + "instance": "5e34f9349f45", "service": "valkey", "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", - "started_at": "2026-10-29T10:13:11.59306872Z", + "started_at": "2026-11-06T07:34:24.1050395Z", "running": true, "exit_code": 0, "memory_bytes": 134217728, @@ -145,44 +145,34 @@ { "type": "volume", "destination": "/data", - "name": "lab3-check-a31f544685_valkey_data" + "name": "lab3-check-2418557caa_valkey_data" } ] } ], "api_tests": "public smoke + 28 student tests passed", + "postgres_pause": { + "history_status": 503, + "history_seconds": 2.008, + "error_code": "unavailable", + "api_instance": "8e14050120c6", + "same_session_and_message_after_unpause": true, + "active_query": { + "status": 503, + "code": "unavailable", + "seconds": 2.002, + "connection_closed": true + }, + "read_write_after_recovery": true + }, "persistence": { "before": [ { - "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", - "instance": "7f6a80b69e32", + "id": "8e14050120c607ce540e41301256a08292ff7b832d7b32682871ad3c1facf560", + "instance": "8e14050120c6", "service": "api", - "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", - "started_at": "2026-10-29T10:13:15.412188555Z", - "running": true, - "exit_code": 0, - "memory_bytes": 268435456, - "nano_cpus": 1000000000, - "published_ports": {}, - "mounts": [ - { - "type": "bind", - "destination": "/run/secrets/cache_url", - "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/database_url", - "name": null - } - ] - }, - { - "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", - "instance": "f973eaec4d52", - "service": "api", - "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", - "started_at": "2026-10-29T10:13:15.325770597Z", + "image": "sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f", + "started_at": "2026-11-06T07:34:27.834668252Z", "running": true, "exit_code": 0, "memory_bytes": 268435456, @@ -202,11 +192,35 @@ ] }, { - "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", - "instance": "2abbb539f04c", + "id": "998bd7b19b8910cbf3e823a15144b6bdbe42a64e15bb9ae602cdbc30a7e3e190", + "instance": "998bd7b19b89", + "service": "api", + "image": "sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f", + "started_at": "2026-11-06T07:34:27.940669294Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "28c0b15ec1e6fa82db624d0742fdeae73aaff4b60bac2ea7ab49a02ff0922f6c", + "instance": "28c0b15ec1e6", "service": "lb", "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", - "started_at": "2026-10-29T10:13:21.014548544Z", + "started_at": "2026-11-06T07:34:33.562351255Z", "running": true, "exit_code": 0, "memory_bytes": 67108864, @@ -215,7 +229,7 @@ "8080/tcp": [ { "HostIp": "127.0.0.1", - "HostPort": "58474" + "HostPort": "64150" } ] }, @@ -228,11 +242,11 @@ ] }, { - "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", - "instance": "653f44c2806d", + "id": "c4dc7a588242d47cc065defca1612a0b818eae805c3643de642f0672ff6e2de4", + "instance": "c4dc7a588242", "service": "postgres", "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", - "started_at": "2026-10-29T10:13:11.598535928Z", + "started_at": "2026-11-06T07:34:24.113683417Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -242,7 +256,7 @@ { "type": "volume", "destination": "/var/lib/postgresql/data", - "name": "lab3-check-a31f544685_postgres_data" + "name": "lab3-check-2418557caa_postgres_data" }, { "type": "bind", @@ -252,11 +266,11 @@ ] }, { - "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", - "instance": "4aca2308d7a0", + "id": "5e34f9349f45e6b95cb35a0f6cf50bce63c8158369c91715140436f2394aaa09", + "instance": "5e34f9349f45", "service": "valkey", "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", - "started_at": "2026-10-29T10:13:11.59306872Z", + "started_at": "2026-11-06T07:34:24.1050395Z", "running": true, "exit_code": 0, "memory_bytes": 134217728, @@ -266,42 +280,18 @@ { "type": "volume", "destination": "/data", - "name": "lab3-check-a31f544685_valkey_data" + "name": "lab3-check-2418557caa_valkey_data" } ] } ], "after": [ { - "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", - "instance": "7f6a80b69e32", + "id": "8e14050120c607ce540e41301256a08292ff7b832d7b32682871ad3c1facf560", + "instance": "8e14050120c6", "service": "api", - "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", - "started_at": "2026-10-29T10:13:35.473955634Z", - "running": true, - "exit_code": 0, - "memory_bytes": 268435456, - "nano_cpus": 1000000000, - "published_ports": {}, - "mounts": [ - { - "type": "bind", - "destination": "/run/secrets/cache_url", - "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/database_url", - "name": null - } - ] - }, - { - "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", - "instance": "f973eaec4d52", - "service": "api", - "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", - "started_at": "2026-10-29T10:13:35.398920009Z", + "image": "sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f", + "started_at": "2026-11-06T07:34:55.237104501Z", "running": true, "exit_code": 0, "memory_bytes": 268435456, @@ -321,11 +311,35 @@ ] }, { - "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", - "instance": "2abbb539f04c", + "id": "998bd7b19b8910cbf3e823a15144b6bdbe42a64e15bb9ae602cdbc30a7e3e190", + "instance": "998bd7b19b89", + "service": "api", + "image": "sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f", + "started_at": "2026-11-06T07:34:55.196763626Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "28c0b15ec1e6fa82db624d0742fdeae73aaff4b60bac2ea7ab49a02ff0922f6c", + "instance": "28c0b15ec1e6", "service": "lb", "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", - "started_at": "2026-10-29T10:13:35.312310009Z", + "started_at": "2026-11-06T07:34:55.112661542Z", "running": true, "exit_code": 0, "memory_bytes": 67108864, @@ -334,7 +348,7 @@ "8080/tcp": [ { "HostIp": "127.0.0.1", - "HostPort": "58474" + "HostPort": "64150" } ] }, @@ -347,35 +361,35 @@ ] }, { - "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", - "instance": "653f44c2806d", + "id": "c4dc7a588242d47cc065defca1612a0b818eae805c3643de642f0672ff6e2de4", + "instance": "c4dc7a588242", "service": "postgres", "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", - "started_at": "2026-10-29T10:13:35.342063384Z", + "started_at": "2026-11-06T07:34:55.144460376Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, "nano_cpus": 1000000000, "published_ports": {}, "mounts": [ - { - "type": "volume", - "destination": "/var/lib/postgresql/data", - "name": "lab3-check-a31f544685_postgres_data" - }, { "type": "bind", "destination": "/run/secrets/postgres_password", "name": null + }, + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab3-check-2418557caa_postgres_data" } ] }, { - "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", - "instance": "4aca2308d7a0", + "id": "5e34f9349f45e6b95cb35a0f6cf50bce63c8158369c91715140436f2394aaa09", + "instance": "5e34f9349f45", "service": "valkey", "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", - "started_at": "2026-10-29T10:13:35.37293605Z", + "started_at": "2026-11-06T07:34:55.173245167Z", "running": true, "exit_code": 0, "memory_bytes": 134217728, @@ -385,7 +399,7 @@ { "type": "volume", "destination": "/data", - "name": "lab3-check-a31f544685_valkey_data" + "name": "lab3-check-2418557caa_valkey_data" } ] } @@ -394,10 +408,10 @@ "idempotency_replay_after_restart": true }, "sigterm": { - "instance": "7f6a80b69e32", + "instance": "8e14050120c6", "unfinished_request": true, - "shutdown_seconds": 5.393, - "new_connections_refused_seconds": 0.167, + "shutdown_seconds": 5.454, + "new_connections_refused_seconds": 0.161, "exit_code": 0, "oom_killed": false }, @@ -419,12 +433,12 @@ "passed": true, "containers_after": [ { - "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", - "instance": "7f6a80b69e32", + "id": "8e14050120c607ce540e41301256a08292ff7b832d7b32682871ad3c1facf560", + "instance": "8e14050120c6", "service": "api", - "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", - "started_at": "2026-10-29T10:13:48.981612751Z", - "running": false, + "image": "sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f", + "started_at": "2026-11-06T07:35:09.013094632Z", + "running": true, "exit_code": 0, "memory_bytes": 268435456, "nano_cpus": 1000000000, @@ -432,23 +446,23 @@ "mounts": [ { "type": "bind", - "destination": "/run/secrets/cache_url", + "destination": "/run/secrets/database_url", "name": null }, { "type": "bind", - "destination": "/run/secrets/database_url", + "destination": "/run/secrets/cache_url", "name": null } ] }, { - "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", - "instance": "f973eaec4d52", + "id": "998bd7b19b8910cbf3e823a15144b6bdbe42a64e15bb9ae602cdbc30a7e3e190", + "instance": "998bd7b19b89", "service": "api", - "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", - "started_at": "2026-10-29T10:13:35.398920009Z", - "running": true, + "image": "sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f", + "started_at": "2026-11-06T07:34:55.196763626Z", + "running": false, "exit_code": 0, "memory_bytes": 268435456, "nano_cpus": 1000000000, @@ -470,188 +484,29 @@ }, "failovers": [ { - "label": "stop 7f6a80b69e32", - "duration_seconds": 16.009, - "successful_requests": 1576, - "successful_writes": 788, + "label": "stop 8e14050120c6", + "duration_seconds": 16.014, + "successful_requests": 1452, + "successful_writes": 726, "errors": 4, - "successful_rps": 98.44, + "successful_rps": 90.67, "first_success_after_action_seconds": { - "POST": 0.014, - "GET": 0.007 + "POST": 0.033, + "GET": 0.044 }, "maximum_success_gap_seconds": { - "POST": 1.957, - "GET": 2.004 + "POST": 2.009, + "GET": 2.003 }, - "action_start_seconds": 1.003, - "action_end_seconds": 1.666, - "p50_ms": 7.813, - "p95_ms": 16.393, + "action_start_seconds": 1.001, + "action_end_seconds": 1.722, + "p50_ms": 9.064, + "p95_ms": 20.562, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": [ - "7f6a80b69e32", - "f973eaec4d52" - ], - "per_second": [ - { - "second": 0, - "successes": 121, - "errors": 0 - }, - { - "second": 1, - "successes": 43, - "errors": 0 - }, - { - "second": 2, - "successes": 0, - "errors": 0 - }, - { - "second": 3, - "successes": 73, - "errors": 4 - }, - { - "second": 4, - "successes": 109, - "errors": 0 - }, - { - "second": 5, - "successes": 113, - "errors": 0 - }, - { - "second": 6, - "successes": 116, - "errors": 0 - }, - { - "second": 7, - "successes": 113, - "errors": 0 - }, - { - "second": 8, - "successes": 118, - "errors": 0 - }, - { - "second": 9, - "successes": 112, - "errors": 0 - }, - { - "second": 10, - "successes": 102, - "errors": 0 - }, - { - "second": 11, - "successes": 102, - "errors": 0 - }, - { - "second": 12, - "successes": 110, - "errors": 0 - }, - { - "second": 13, - "successes": 116, - "errors": 0 - }, - { - "second": 14, - "successes": 112, - "errors": 0 - }, - { - "second": 15, - "successes": 116, - "errors": 0 - }, - { - "second": 16, - "successes": 0, - "errors": 0 - } - ], - "error_samples": [ - [ - 3.329044791999877, - "GET", - "GET /api/v1/chats/76736023-9461-41c6-87f7-2286c6b1b7b3/messages: connection/TLS failure (TimeoutError)" - ], - [ - 3.363858082999741, - "POST", - "POST /api/v1/chats/76736023-9461-41c6-87f7-2286c6b1b7b3/messages: connection/TLS failure (TimeoutError)" - ], - [ - 3.367696166999849, - "POST", - "POST /api/v1/chats/76736023-9461-41c6-87f7-2286c6b1b7b3/messages: connection/TLS failure (TimeoutError)" - ], - [ - 3.4346232499997313, - "GET", - "GET /api/v1/chats/76736023-9461-41c6-87f7-2286c6b1b7b3/messages: connection/TLS failure (TimeoutError)" - ] - ], - "action": { - "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", - "instance": "7f6a80b69e32", - "service": "api", - "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", - "started_at": "2026-10-29T10:14:23.665074878Z", - "running": false, - "exit_code": 0, - "memory_bytes": 268435456, - "nano_cpus": 1000000000, - "published_ports": {}, - "mounts": [ - { - "type": "bind", - "destination": "/run/secrets/cache_url", - "name": null - }, - { - "type": "bind", - "destination": "/run/secrets/database_url", - "name": null - } - ] - } - }, - { - "label": "stop f973eaec4d52", - "duration_seconds": 16.015, - "successful_requests": 1574, - "successful_writes": 786, - "errors": 4, - "successful_rps": 98.28, - "first_success_after_action_seconds": { - "POST": 0.002, - "GET": 0.008 - }, - "maximum_success_gap_seconds": { - "POST": 2.006, - "GET": 2.006 - }, - "action_start_seconds": 1.005, - "action_end_seconds": 1.713, - "p50_ms": 7.98, - "p95_ms": 17.217, - "confirmed_data_preserved": true, - "duplicate_messages": 0, - "instances": [ - "7f6a80b69e32", - "f973eaec4d52" + "8e14050120c6", + "998bd7b19b89" ], "per_second": [ { @@ -661,7 +516,7 @@ }, { "second": 1, - "successes": 43, + "successes": 40, "errors": 0 }, { @@ -671,62 +526,62 @@ }, { "second": 3, - "successes": 71, + "successes": 68, "errors": 4 }, { "second": 4, - "successes": 117, + "successes": 106, "errors": 0 }, { "second": 5, - "successes": 115, + "successes": 96, "errors": 0 }, { "second": 6, - "successes": 109, + "successes": 90, "errors": 0 }, { "second": 7, - "successes": 116, + "successes": 104, "errors": 0 }, { "second": 8, - "successes": 109, + "successes": 96, "errors": 0 }, { "second": 9, - "successes": 118, + "successes": 107, "errors": 0 }, { "second": 10, - "successes": 115, + "successes": 102, "errors": 0 }, { "second": 11, - "successes": 115, + "successes": 111, "errors": 0 }, { "second": 12, - "successes": 109, + "successes": 96, "errors": 0 }, { "second": 13, - "successes": 105, + "successes": 106, "errors": 0 }, { "second": 14, - "successes": 113, + "successes": 112, "errors": 0 }, { @@ -736,38 +591,197 @@ }, { "second": 16, - "successes": 3, + "successes": 2, "errors": 0 } ], "error_samples": [ [ - 3.3455931250000503, + 3.3445420839998405, "POST", - "POST /api/v1/chats/aba5162e-1b67-4157-9f13-85a2427d7f80/messages: connection/TLS failure (TimeoutError)" + "POST /api/v1/chats/016772af-0bbf-4860-98c2-6a847c3590dd/messages: connection/TLS failure (TimeoutError)" ], [ - 3.349944458000209, + 3.3737433339993004, "POST", - "POST /api/v1/chats/aba5162e-1b67-4157-9f13-85a2427d7f80/messages: connection/TLS failure (TimeoutError)" + "POST /api/v1/chats/016772af-0bbf-4860-98c2-6a847c3590dd/messages: connection/TLS failure (TimeoutError)" ], [ - 3.3524864160003744, - "POST", - "POST /api/v1/chats/aba5162e-1b67-4157-9f13-85a2427d7f80/messages: connection/TLS failure (TimeoutError)" - ], - [ - 3.41803337500005, + 3.38191112500499, "GET", - "GET /api/v1/chats/aba5162e-1b67-4157-9f13-85a2427d7f80/messages: connection/TLS failure (TimeoutError)" + "GET /api/v1/chats/016772af-0bbf-4860-98c2-6a847c3590dd/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.4136469999939436, + "GET", + "GET /api/v1/chats/016772af-0bbf-4860-98c2-6a847c3590dd/messages: connection/TLS failure (TimeoutError)" ] ], "action": { - "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", - "instance": "f973eaec4d52", + "id": "8e14050120c607ce540e41301256a08292ff7b832d7b32682871ad3c1facf560", + "instance": "8e14050120c6", "service": "api", - "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", - "started_at": "2026-10-29T10:13:35.398920009Z", + "image": "sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f", + "started_at": "2026-11-06T07:35:09.013094632Z", + "running": false, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + } + }, + { + "label": "stop 998bd7b19b89", + "duration_seconds": 16.015, + "successful_requests": 1504, + "successful_writes": 751, + "errors": 4, + "successful_rps": 93.91, + "first_success_after_action_seconds": { + "POST": 0.015, + "GET": 0.021 + }, + "maximum_success_gap_seconds": { + "POST": 1.968, + "GET": 1.972 + }, + "action_start_seconds": 1.011, + "action_end_seconds": 1.701, + "p50_ms": 8.418, + "p95_ms": 16.635, + "confirmed_data_preserved": true, + "duplicate_messages": 0, + "instances": [ + "8e14050120c6", + "998bd7b19b89" + ], + "per_second": [ + { + "second": 0, + "successes": 118, + "errors": 0 + }, + { + "second": 1, + "successes": 41, + "errors": 0 + }, + { + "second": 2, + "successes": 0, + "errors": 0 + }, + { + "second": 3, + "successes": 59, + "errors": 4 + }, + { + "second": 4, + "successes": 113, + "errors": 0 + }, + { + "second": 5, + "successes": 105, + "errors": 0 + }, + { + "second": 6, + "successes": 108, + "errors": 0 + }, + { + "second": 7, + "successes": 109, + "errors": 0 + }, + { + "second": 8, + "successes": 102, + "errors": 0 + }, + { + "second": 9, + "successes": 105, + "errors": 0 + }, + { + "second": 10, + "successes": 103, + "errors": 0 + }, + { + "second": 11, + "successes": 109, + "errors": 0 + }, + { + "second": 12, + "successes": 108, + "errors": 0 + }, + { + "second": 13, + "successes": 108, + "errors": 0 + }, + { + "second": 14, + "successes": 112, + "errors": 0 + }, + { + "second": 15, + "successes": 104, + "errors": 0 + }, + { + "second": 16, + "successes": 0, + "errors": 0 + } + ], + "error_samples": [ + [ + 3.3396142910060007, + "POST", + "POST /api/v1/chats/d6b57d5f-7fde-4c24-a854-44f01806a1f6/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.3734452909993706, + "POST", + "POST /api/v1/chats/d6b57d5f-7fde-4c24-a854-44f01806a1f6/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.383555000007618, + "POST", + "POST /api/v1/chats/d6b57d5f-7fde-4c24-a854-44f01806a1f6/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.4469545410102, + "GET", + "GET /api/v1/chats/d6b57d5f-7fde-4c24-a854-44f01806a1f6/messages: connection/TLS failure (TimeoutError)" + ] + ], + "action": { + "id": "998bd7b19b8910cbf3e823a15144b6bdbe42a64e15bb9ae602cdbc30a7e3e190", + "instance": "998bd7b19b89", + "service": "api", + "image": "sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f", + "started_at": "2026-11-06T07:35:43.846525885Z", "running": false, "exit_code": 0, "memory_bytes": 268435456, @@ -790,188 +804,183 @@ ], "rolling_update": { "label": "sequential API replacement", - "duration_seconds": 23.774, - "successful_requests": 2131, - "successful_writes": 1063, - "errors": 11, - "successful_rps": 89.64, + "duration_seconds": 22.757, + "successful_requests": 2072, + "successful_writes": 1037, + "errors": 8, + "successful_rps": 91.05, "first_success_after_action_seconds": { - "POST": 0.018, - "GET": 0.004 + "POST": 0.019, + "GET": 0.025 }, "maximum_success_gap_seconds": { - "POST": 1.059, - "GET": 2.034 + "POST": 1.007, + "GET": 1.074 }, - "action_start_seconds": 1.005, - "action_end_seconds": 19.63, - "p50_ms": 7.723, - "p95_ms": 13.565, + "action_start_seconds": 1.008, + "action_end_seconds": 19.74, + "p50_ms": 8.068, + "p95_ms": 14.446, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": [ - "244bf0b910eb", - "7f6a80b69e32", - "ca9c7a52da66", - "f973eaec4d52" + "1dde8575f8f4", + "27f47ac686fc", + "8e14050120c6", + "998bd7b19b89" ], "per_second": [ { "second": 0, - "successes": 121, + "successes": 112, "errors": 0 }, { "second": 1, - "successes": 116, + "successes": 106, "errors": 0 }, { "second": 2, - "successes": 109, + "successes": 108, "errors": 0 }, { "second": 3, - "successes": 104, + "successes": 107, "errors": 0 }, { "second": 4, - "successes": 114, + "successes": 97, "errors": 0 }, { "second": 5, - "successes": 33, - "errors": 0 - }, - { - "second": 6, - "successes": 60, + "successes": 10, "errors": 4 }, + { + "second": 6, + "successes": 22, + "errors": 0 + }, { "second": 7, - "successes": 105, + "successes": 86, "errors": 0 }, { "second": 8, - "successes": 117, + "successes": 115, "errors": 0 }, { "second": 9, - "successes": 119, + "successes": 116, "errors": 0 }, { "second": 10, - "successes": 116, + "successes": 108, "errors": 0 }, { "second": 11, - "successes": 115, + "successes": 111, "errors": 0 }, { "second": 12, - "successes": 28, + "successes": 19, "errors": 0 }, { "second": 13, - "successes": 5, + "successes": 3, "errors": 4 }, { "second": 14, - "successes": 74, + "successes": 91, "errors": 0 }, { "second": 15, - "successes": 120, + "successes": 111, "errors": 0 }, { "second": 16, - "successes": 119, + "successes": 112, "errors": 0 }, { "second": 17, - "successes": 115, + "successes": 113, "errors": 0 }, { "second": 18, - "successes": 114, + "successes": 108, "errors": 0 }, { "second": 19, - "successes": 120, + "successes": 111, "errors": 0 }, { "second": 20, - "successes": 116, + "successes": 112, "errors": 0 }, { "second": 21, - "successes": 88, + "successes": 109, "errors": 0 }, { "second": 22, - "successes": 0, + "successes": 85, "errors": 0 - }, - { - "second": 23, - "successes": 3, - "errors": 3 } ], "error_samples": [ [ - 6.263843457999883, - "POST", - "POST /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [201], received 503" - ], - [ - 6.297545500000069, - "POST", - "POST /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [201], received 503" - ], - [ - 6.300835125000049, - "POST", - "POST /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [201], received 503" - ], - [ - 6.369535666000047, + 5.859570375003386, "GET", - "GET /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [200], received 503" + "GET /api/v1/chats/7e626360-a780-4ddc-b215-907e3e6b85af/messages: expected [200], received 503" ], [ - 13.222746333000032, + 5.876960459005204, + "GET", + "GET /api/v1/chats/7e626360-a780-4ddc-b215-907e3e6b85af/messages: expected [200], received 503" + ], + [ + 5.907214917009696, + "GET", + "GET /api/v1/chats/7e626360-a780-4ddc-b215-907e3e6b85af/messages: expected [200], received 503" + ], + [ + 5.91258333400765, + "GET", + "GET /api/v1/chats/7e626360-a780-4ddc-b215-907e3e6b85af/messages: expected [200], received 503" + ], + [ + 13.1466267920041, "POST", - "POST /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [201], received 503" + "POST /api/v1/chats/7e626360-a780-4ddc-b215-907e3e6b85af/messages: expected [201], received 503" ] ], "action": { "before": [ { - "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", - "instance": "7f6a80b69e32", + "id": "8e14050120c607ce540e41301256a08292ff7b832d7b32682871ad3c1facf560", + "instance": "8e14050120c6", "service": "api", - "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", - "started_at": "2026-10-29T10:14:47.987597625Z", + "image": "sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f", + "started_at": "2026-11-06T07:36:08.104300215Z", "running": true, "exit_code": 0, "memory_bytes": 268435456, @@ -980,22 +989,22 @@ "mounts": [ { "type": "bind", - "destination": "/run/secrets/cache_url", + "destination": "/run/secrets/database_url", "name": null }, { "type": "bind", - "destination": "/run/secrets/database_url", + "destination": "/run/secrets/cache_url", "name": null } ] }, { - "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", - "instance": "f973eaec4d52", + "id": "998bd7b19b8910cbf3e823a15144b6bdbe42a64e15bb9ae602cdbc30a7e3e190", + "instance": "998bd7b19b89", "service": "api", - "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", - "started_at": "2026-10-29T10:15:11.976704845Z", + "image": "sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f", + "started_at": "2026-11-06T07:36:32.161170421Z", "running": true, "exit_code": 0, "memory_bytes": 268435456, @@ -1004,22 +1013,22 @@ "mounts": [ { "type": "bind", - "destination": "/run/secrets/cache_url", + "destination": "/run/secrets/database_url", "name": null }, { "type": "bind", - "destination": "/run/secrets/database_url", + "destination": "/run/secrets/cache_url", "name": null } ] }, { - "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", - "instance": "2abbb539f04c", + "id": "28c0b15ec1e6fa82db624d0742fdeae73aaff4b60bac2ea7ab49a02ff0922f6c", + "instance": "28c0b15ec1e6", "service": "lb", "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", - "started_at": "2026-10-29T10:13:35.312310009Z", + "started_at": "2026-11-06T07:34:55.112661542Z", "running": true, "exit_code": 0, "memory_bytes": 67108864, @@ -1028,7 +1037,7 @@ "8080/tcp": [ { "HostIp": "127.0.0.1", - "HostPort": "58474" + "HostPort": "64150" } ] }, @@ -1041,11 +1050,11 @@ ] }, { - "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", - "instance": "653f44c2806d", + "id": "c4dc7a588242d47cc065defca1612a0b818eae805c3643de642f0672ff6e2de4", + "instance": "c4dc7a588242", "service": "postgres", "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", - "started_at": "2026-10-29T10:14:01.922362174Z", + "started_at": "2026-11-06T07:35:22.154406263Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -1055,7 +1064,7 @@ { "type": "volume", "destination": "/var/lib/postgresql/data", - "name": "lab3-check-a31f544685_postgres_data" + "name": "lab3-check-2418557caa_postgres_data" }, { "type": "bind", @@ -1065,11 +1074,11 @@ ] }, { - "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", - "instance": "4aca2308d7a0", + "id": "5e34f9349f45e6b95cb35a0f6cf50bce63c8158369c91715140436f2394aaa09", + "instance": "5e34f9349f45", "service": "valkey", "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", - "started_at": "2026-10-29T10:13:56.000628546Z", + "started_at": "2026-11-06T07:35:16.191112511Z", "running": true, "exit_code": 0, "memory_bytes": 134217728, @@ -1079,18 +1088,18 @@ { "type": "volume", "destination": "/data", - "name": "lab3-check-a31f544685_valkey_data" + "name": "lab3-check-2418557caa_valkey_data" } ] } ], "after": [ { - "id": "ca9c7a52da66ca3c5ffad5cc6668a0d6d60876862683967b6d5f8e438eb630e0", - "instance": "ca9c7a52da66", + "id": "1dde8575f8f47901f0885407b1ee7ea14e05413b974ea309a6a6117bbc5d46ee", + "instance": "1dde8575f8f4", "service": "api", - "image": "sha256:3e102cd9a2c1e81e95aed4a0d4de1ac94352370595fced350257813f7a07bef8", - "started_at": "2026-10-29T10:15:24.374360503Z", + "image": "sha256:6b3faf75764ac8f93967166e18995765c3c9899adae08c10b0df375a8a871db1", + "started_at": "2026-11-06T07:36:44.022233593Z", "running": true, "exit_code": 0, "memory_bytes": 268435456, @@ -1110,11 +1119,11 @@ ] }, { - "id": "244bf0b910eb2fa37937f8f0756a1162e9b339df076f09f4cff90b5c357037a6", - "instance": "244bf0b910eb", + "id": "27f47ac686fc84ad11a443c49cb04a8e540da3698b7fac770ef83d96bea54747", + "instance": "27f47ac686fc", "service": "api", - "image": "sha256:3e102cd9a2c1e81e95aed4a0d4de1ac94352370595fced350257813f7a07bef8", - "started_at": "2026-10-29T10:15:31.325585882Z", + "image": "sha256:6b3faf75764ac8f93967166e18995765c3c9899adae08c10b0df375a8a871db1", + "started_at": "2026-11-06T07:36:51.294172721Z", "running": true, "exit_code": 0, "memory_bytes": 268435456, @@ -1134,11 +1143,11 @@ ] }, { - "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", - "instance": "2abbb539f04c", + "id": "28c0b15ec1e6fa82db624d0742fdeae73aaff4b60bac2ea7ab49a02ff0922f6c", + "instance": "28c0b15ec1e6", "service": "lb", "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", - "started_at": "2026-10-29T10:13:35.312310009Z", + "started_at": "2026-11-06T07:34:55.112661542Z", "running": true, "exit_code": 0, "memory_bytes": 67108864, @@ -1147,7 +1156,7 @@ "8080/tcp": [ { "HostIp": "127.0.0.1", - "HostPort": "58474" + "HostPort": "64150" } ] }, @@ -1160,11 +1169,11 @@ ] }, { - "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", - "instance": "653f44c2806d", + "id": "c4dc7a588242d47cc065defca1612a0b818eae805c3643de642f0672ff6e2de4", + "instance": "c4dc7a588242", "service": "postgres", "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", - "started_at": "2026-10-29T10:14:01.922362174Z", + "started_at": "2026-11-06T07:35:22.154406263Z", "running": true, "exit_code": 0, "memory_bytes": 536870912, @@ -1174,7 +1183,7 @@ { "type": "volume", "destination": "/var/lib/postgresql/data", - "name": "lab3-check-a31f544685_postgres_data" + "name": "lab3-check-2418557caa_postgres_data" }, { "type": "bind", @@ -1184,11 +1193,11 @@ ] }, { - "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", - "instance": "4aca2308d7a0", + "id": "5e34f9349f45e6b95cb35a0f6cf50bce63c8158369c91715140436f2394aaa09", + "instance": "5e34f9349f45", "service": "valkey", "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", - "started_at": "2026-10-29T10:13:56.000628546Z", + "started_at": "2026-11-06T07:35:16.191112511Z", "running": true, "exit_code": 0, "memory_bytes": 134217728, @@ -1198,7 +1207,7 @@ { "type": "volume", "destination": "/data", - "name": "lab3-check-a31f544685_valkey_data" + "name": "lab3-check-2418557caa_valkey_data" } ] } diff --git a/evidence/lab3-checks.txt b/evidence/lab3-checks.txt index 27ec14e..051aaa2 100644 --- a/evidence/lab3-checks.txt +++ b/evidence/lab3-checks.txt @@ -1,4 +1,4 @@ -Command: ./scripts/check-docker.sh; project: lab3-check-a31f544685 +Command: ./scripts/check-docker.sh; project: lab3-check-2418557caa Docker: 29.0.1 aarch64 CPUs=10 RAM=8217448448 Local .env and secrets are ready; values are not printed #1 [internal] load local bake definitions @@ -9,117 +9,118 @@ Local .env and secrets are ready; values are not printed #2 transferring dockerfile: 733B done #2 DONE 0.0s -#3 [api internal] load metadata for docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#3 [migrate internal] load metadata for docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 #3 DONE 0.0s -#4 [api internal] load .dockerignore +#4 [migrate internal] load .dockerignore #4 transferring context: 167B done #4 DONE 0.0s -#5 [migrate internal] load build context +#5 [api internal] load build context #5 transferring context: 809B done #5 DONE 0.0s -#6 [migrate 1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 -#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 done +#6 [api 1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 0.0s done #6 DONE 0.0s -#7 [migrate 2/8] WORKDIR /app +#7 [api 6/8] COPY src/ ./src/ #7 CACHED -#8 [migrate 3/8] COPY requirements.txt . +#8 [api 3/8] COPY requirements.txt . #8 CACHED -#9 [migrate 4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#9 [api 7/8] COPY migrations/ ./migrations/ #9 CACHED -#10 [migrate 6/8] COPY src/ ./src/ +#10 [api 4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check #10 CACHED -#11 [migrate 7/8] COPY migrations/ ./migrations/ +#11 [api 2/8] WORKDIR /app #11 CACHED -#12 [migrate 5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#12 [api 5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app #12 CACHED -#13 [migrate 8/8] COPY research/ ./research/ +#13 [api 8/8] COPY research/ ./research/ #13 CACHED #14 [api] exporting to image #14 exporting layers done -#14 exporting manifest sha256:06163e45d3676c7cf4f7a1c7bfe108ed3b98559ebb485c7a09d1e87e6f418d17 done -#14 exporting config sha256:3fed7460e9510eef7b2928f8d600d06d5d65329f74d820beac13809476b11398 done -#14 exporting attestation manifest sha256:352319cde788733f2f237a2a78204630c542e7ca906cff14c902046e8c1d2e5b 0.0s done -#14 exporting manifest list sha256:848e0c76f1678f35dd1b483e7c54552466999551ba7e77c0062bf346671e2902 done +#14 exporting manifest sha256:f5b0ba5ed30baa51240061b64983bd1e0be184515995717d9d42c0f303b28386 0.0s done +#14 exporting config sha256:ba8834f54a12be5f335a7d34fccbeb70b0c1b7f1535c33e1a8bf0510bedb4d72 done +#14 exporting attestation manifest sha256:fc530f9da54cb9b04d0e2eb260225df8d61746a14f47eca74e1035ba073c23a6 0.0s done +#14 exporting manifest list sha256:2af1c545ac849373a709167b3fda775bb89e3be8b0efd868844ed36ca154b1e6 done +#14 naming to docker.io/library/mtusi-messenger:lab3 #14 naming to docker.io/library/mtusi-messenger:lab3 done #14 unpacking to docker.io/library/mtusi-messenger:lab3 done #14 DONE 0.1s #15 [migrate] exporting to image #15 exporting layers done -#15 exporting manifest sha256:a59568d34e2c5e8b4d23308342601fdaf8c07e78ec3065878679a067c082e952 done -#15 exporting config sha256:b67b805ba1b7dd90f0a3e8444455488fe8c7f4118f14d671501dd63e85b59c97 done -#15 exporting attestation manifest sha256:b1853c07ddac46c4b398b475052f334aa2753c2fe971e0e4fb2517b0e3e13656 0.0s done -#15 exporting manifest list sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb done +#15 exporting manifest sha256:b9475efa88de4b71344df73089a66ddd577d05384dce92c62b755b04f74934cc 0.0s done +#15 exporting config sha256:18b572ee6ee932396009b6354f40403917b55bd3d69920671450baff5304f07f done +#15 exporting attestation manifest sha256:a9debc4af71e6876f412c529b350ec078d267c239bd643e509597224309139ed 0.0s done +#15 exporting manifest list sha256:5d4b6704cc0e11813a73dc8a21d040fd1fa0a7155e40e221bd7ff35297bd729f done #15 naming to docker.io/library/mtusi-messenger:lab3 done #15 unpacking to docker.io/library/mtusi-messenger:lab3 done #15 DONE 0.1s -#16 [api] resolving provenance for metadata file +#16 [migrate] resolving provenance for metadata file #16 DONE 0.0s -#17 [migrate] resolving provenance for metadata file +#17 [api] resolving provenance for metadata file #17 DONE 0.0s mtusi-messenger:lab3 Built mtusi-messenger:lab3 Built - Network lab3-check-a31f544685_default Creating - Network lab3-check-a31f544685_default Created - Volume lab3-check-a31f544685_postgres_data Creating - Volume lab3-check-a31f544685_postgres_data Created - Volume lab3-check-a31f544685_valkey_data Creating - Volume lab3-check-a31f544685_valkey_data Created - Container lab3-check-a31f544685-valkey-1 Creating - Container lab3-check-a31f544685-postgres-1 Creating - Container lab3-check-a31f544685-valkey-1 Created - Container lab3-check-a31f544685-postgres-1 Created - Container lab3-check-a31f544685-migrate-1 Creating - Container lab3-check-a31f544685-migrate-1 Created - Container lab3-check-a31f544685-api-2 Creating - Container lab3-check-a31f544685-api-1 Creating - Container lab3-check-a31f544685-api-1 Created - Container lab3-check-a31f544685-api-2 Created - Container lab3-check-a31f544685-lb-1 Creating - Container lab3-check-a31f544685-lb-1 Created - Container lab3-check-a31f544685-valkey-1 Starting - Container lab3-check-a31f544685-postgres-1 Starting - Container lab3-check-a31f544685-valkey-1 Started - Container lab3-check-a31f544685-postgres-1 Started - Container lab3-check-a31f544685-postgres-1 Waiting - Container lab3-check-a31f544685-postgres-1 Healthy - Container lab3-check-a31f544685-migrate-1 Starting - Container lab3-check-a31f544685-migrate-1 Started - Container lab3-check-a31f544685-migrate-1 Waiting - Container lab3-check-a31f544685-postgres-1 Waiting - Container lab3-check-a31f544685-valkey-1 Waiting - Container lab3-check-a31f544685-valkey-1 Healthy - Container lab3-check-a31f544685-postgres-1 Healthy - Container lab3-check-a31f544685-migrate-1 Exited - Container lab3-check-a31f544685-api-2 Starting - Container lab3-check-a31f544685-api-2 Started - Container lab3-check-a31f544685-api-1 Starting - Container lab3-check-a31f544685-api-1 Started - Container lab3-check-a31f544685-api-2 Waiting - Container lab3-check-a31f544685-api-1 Waiting - Container lab3-check-a31f544685-api-2 Healthy - Container lab3-check-a31f544685-api-1 Healthy - Container lab3-check-a31f544685-lb-1 Starting - Container lab3-check-a31f544685-lb-1 Started -API ready through LB: 7f6a80b69e32, f973eaec4d52 + Network lab3-check-2418557caa_default Creating + Network lab3-check-2418557caa_default Created + Volume lab3-check-2418557caa_postgres_data Creating + Volume lab3-check-2418557caa_postgres_data Created + Volume lab3-check-2418557caa_valkey_data Creating + Volume lab3-check-2418557caa_valkey_data Created + Container lab3-check-2418557caa-valkey-1 Creating + Container lab3-check-2418557caa-postgres-1 Creating + Container lab3-check-2418557caa-valkey-1 Created + Container lab3-check-2418557caa-postgres-1 Created + Container lab3-check-2418557caa-migrate-1 Creating + Container lab3-check-2418557caa-migrate-1 Created + Container lab3-check-2418557caa-api-2 Creating + Container lab3-check-2418557caa-api-1 Creating + Container lab3-check-2418557caa-api-2 Created + Container lab3-check-2418557caa-api-1 Created + Container lab3-check-2418557caa-lb-1 Creating + Container lab3-check-2418557caa-lb-1 Created + Container lab3-check-2418557caa-valkey-1 Starting + Container lab3-check-2418557caa-postgres-1 Starting + Container lab3-check-2418557caa-postgres-1 Started + Container lab3-check-2418557caa-postgres-1 Waiting + Container lab3-check-2418557caa-valkey-1 Started + Container lab3-check-2418557caa-postgres-1 Healthy + Container lab3-check-2418557caa-migrate-1 Starting + Container lab3-check-2418557caa-migrate-1 Started + Container lab3-check-2418557caa-valkey-1 Waiting + Container lab3-check-2418557caa-migrate-1 Waiting + Container lab3-check-2418557caa-postgres-1 Waiting + Container lab3-check-2418557caa-valkey-1 Healthy + Container lab3-check-2418557caa-postgres-1 Healthy + Container lab3-check-2418557caa-migrate-1 Exited + Container lab3-check-2418557caa-api-1 Starting + Container lab3-check-2418557caa-api-1 Started + Container lab3-check-2418557caa-api-2 Starting + Container lab3-check-2418557caa-api-2 Started + Container lab3-check-2418557caa-api-1 Waiting + Container lab3-check-2418557caa-api-2 Waiting + Container lab3-check-2418557caa-api-1 Healthy + Container lab3-check-2418557caa-api-2 Healthy + Container lab3-check-2418557caa-lb-1 Starting + Container lab3-check-2418557caa-lb-1 Started +API ready through LB: 8e14050120c6, 998bd7b19b89 PASS: 2 real API containers, CPU/RAM limits, API non-root and no public ports; only LB published -Проверяем сервер: http://127.0.0.1:58474 +Проверяем сервер: http://127.0.0.1:64150 /Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python scripts/check_contract.py PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. -/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:58474" +/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:64150" PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. test_chat_access (test_api.ApiTests.test_chat_access) ... ok test_chat_pagination_and_cursor_owner (test_api.ApiTests.test_chat_pagination_and_cursor_owner) ... ok @@ -151,90 +152,97 @@ test_unicode_password_with_colon_and_private_response (test_sessions.SessionTest test_wrong_password_and_basic_only_for_login (test_sessions.SessionTests.test_wrong_password_and_basic_only_for_login) ... ok ---------------------------------------------------------------------- -Ran 28 tests in 6.783s +Ran 28 tests in 6.931s OK PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages. - Container lab3-check-a31f544685-postgres-1 Running - Container lab3-check-a31f544685-postgres-1 Running + Container lab3-check-2418557caa-postgres-1 Running + Container lab3-check-2418557caa-postgres-1 Running PASS: migrations only in one-shot service; repeated execution preserves data -/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:58474" + Container lab3-check-2418557caa-postgres-1 Paused + Container lab3-check-2418557caa-postgres-1 Unpaused +API ready through LB: 8e14050120c6, 998bd7b19b89 + Container lab3-check-2418557caa-postgres-1 Paused + Container lab3-check-2418557caa-postgres-1 Unpaused +API ready through LB: 8e14050120c6, 998bd7b19b89 +PASS: PostgreSQL pause: {"history_status": 503, "history_seconds": 2.008, "error_code": "unavailable", "api_instance": "8e14050120c6", "same_session_and_message_after_unpause": true, "active_query": {"status": 503, "code": "unavailable", "seconds": 2.002, "connection_closed": true}, "read_write_after_recovery": true} +/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:64150" Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. PASS: both API, LB, PostgreSQL, Valkey actually restarted; session and confirmed data preserved -PASS: SIGTERM with unfinished request: {"instance": "7f6a80b69e32", "unfinished_request": true, "shutdown_seconds": 5.393, "new_connections_refused_seconds": 0.167, "exit_code": 0, "oom_killed": false} - Container lab3-check-a31f544685-postgres-1 Waiting - Container lab3-check-a31f544685-postgres-1 Healthy - Container lab3-check-a31f544685-migrate-1 Starting - Container lab3-check-a31f544685-migrate-1 Started - Container lab3-check-a31f544685-valkey-1 Waiting - Container lab3-check-a31f544685-migrate-1 Waiting - Container lab3-check-a31f544685-postgres-1 Waiting - Container lab3-check-a31f544685-valkey-1 Healthy - Container lab3-check-a31f544685-postgres-1 Healthy - Container lab3-check-a31f544685-migrate-1 Exited - Container lab3-check-a31f544685-api-1 Starting - Container lab3-check-a31f544685-api-1 Started -API ready through LB: 7f6a80b69e32, f973eaec4d52 - Container lab3-check-a31f544685-valkey-1 Stopping - Container lab3-check-a31f544685-valkey-1 Stopped +PASS: SIGTERM with unfinished request: {"instance": "8e14050120c6", "unfinished_request": true, "shutdown_seconds": 5.454, "new_connections_refused_seconds": 0.161, "exit_code": 0, "oom_killed": false} + Container lab3-check-2418557caa-postgres-1 Waiting + Container lab3-check-2418557caa-postgres-1 Healthy + Container lab3-check-2418557caa-migrate-1 Starting + Container lab3-check-2418557caa-migrate-1 Started + Container lab3-check-2418557caa-migrate-1 Waiting + Container lab3-check-2418557caa-postgres-1 Waiting + Container lab3-check-2418557caa-valkey-1 Waiting + Container lab3-check-2418557caa-postgres-1 Healthy + Container lab3-check-2418557caa-valkey-1 Healthy + Container lab3-check-2418557caa-migrate-1 Exited + Container lab3-check-2418557caa-api-1 Starting + Container lab3-check-2418557caa-api-1 Started +API ready through LB: 8e14050120c6, 998bd7b19b89 + Container lab3-check-2418557caa-valkey-1 Stopping + Container lab3-check-2418557caa-valkey-1 Stopped PASS: valkey outage: API live=200, ready=503, LB ready=503 - Container lab3-check-a31f544685-valkey-1 Starting - Container lab3-check-a31f544685-valkey-1 Started -API ready through LB: 7f6a80b69e32, f973eaec4d52 - Container lab3-check-a31f544685-postgres-1 Stopping - Container lab3-check-a31f544685-postgres-1 Stopped + Container lab3-check-2418557caa-valkey-1 Starting + Container lab3-check-2418557caa-valkey-1 Started +API ready through LB: 8e14050120c6, 998bd7b19b89 + Container lab3-check-2418557caa-postgres-1 Stopping + Container lab3-check-2418557caa-postgres-1 Stopped PASS: postgres outage: API live=200, ready=503, LB ready=503 - Container lab3-check-a31f544685-postgres-1 Starting - Container lab3-check-a31f544685-postgres-1 Started -API ready through LB: 7f6a80b69e32, f973eaec4d52 -/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:58474" + Container lab3-check-2418557caa-postgres-1 Starting + Container lab3-check-2418557caa-postgres-1 Started +API ready through LB: 8e14050120c6, 998bd7b19b89 +/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:64150" Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 113, "failed_attempts": 1, "first_recovery_seconds": 2.452} +{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 1, "first_recovery_seconds": 2.459} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. - Container lab3-check-a31f544685-postgres-1 Waiting - Container lab3-check-a31f544685-postgres-1 Healthy - Container lab3-check-a31f544685-migrate-1 Starting - Container lab3-check-a31f544685-migrate-1 Started - Container lab3-check-a31f544685-postgres-1 Waiting - Container lab3-check-a31f544685-valkey-1 Waiting - Container lab3-check-a31f544685-migrate-1 Waiting - Container lab3-check-a31f544685-postgres-1 Healthy - Container lab3-check-a31f544685-valkey-1 Healthy - Container lab3-check-a31f544685-migrate-1 Exited - Container lab3-check-a31f544685-api-1 Starting - Container lab3-check-a31f544685-api-1 Started -API ready through LB: 7f6a80b69e32, f973eaec4d52 -Stopped API: 7f6a80b69e32 container=7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6 -{"label": "stop 7f6a80b69e32", "duration_seconds": 16.009, "successful_requests": 1576, "successful_writes": 788, "errors": 4, "successful_rps": 98.44, "first_success_after_action_seconds": {"POST": 0.014, "GET": 0.007}, "maximum_success_gap_seconds": {"POST": 1.957, "GET": 2.004}, "action_start_seconds": 1.003, "action_end_seconds": 1.666, "p50_ms": 7.813, "p95_ms": 16.393, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["7f6a80b69e32", "f973eaec4d52"]} - Container lab3-check-a31f544685-postgres-1 Waiting - Container lab3-check-a31f544685-postgres-1 Healthy - Container lab3-check-a31f544685-migrate-1 Starting - Container lab3-check-a31f544685-migrate-1 Started - Container lab3-check-a31f544685-postgres-1 Waiting - Container lab3-check-a31f544685-valkey-1 Waiting - Container lab3-check-a31f544685-migrate-1 Waiting - Container lab3-check-a31f544685-valkey-1 Healthy - Container lab3-check-a31f544685-postgres-1 Healthy - Container lab3-check-a31f544685-migrate-1 Exited - Container lab3-check-a31f544685-api-1 Starting - Container lab3-check-a31f544685-api-1 Started -API ready through LB: 7f6a80b69e32, f973eaec4d52 -Stopped API: f973eaec4d52 container=f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b -{"label": "stop f973eaec4d52", "duration_seconds": 16.015, "successful_requests": 1574, "successful_writes": 786, "errors": 4, "successful_rps": 98.28, "first_success_after_action_seconds": {"POST": 0.002, "GET": 0.008}, "maximum_success_gap_seconds": {"POST": 2.006, "GET": 2.006}, "action_start_seconds": 1.005, "action_end_seconds": 1.713, "p50_ms": 7.98, "p95_ms": 17.217, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["7f6a80b69e32", "f973eaec4d52"]} - Container lab3-check-a31f544685-postgres-1 Waiting - Container lab3-check-a31f544685-postgres-1 Healthy - Container lab3-check-a31f544685-migrate-1 Starting - Container lab3-check-a31f544685-migrate-1 Started - Container lab3-check-a31f544685-postgres-1 Waiting - Container lab3-check-a31f544685-valkey-1 Waiting - Container lab3-check-a31f544685-migrate-1 Waiting - Container lab3-check-a31f544685-valkey-1 Healthy - Container lab3-check-a31f544685-postgres-1 Healthy - Container lab3-check-a31f544685-migrate-1 Exited - Container lab3-check-a31f544685-api-2 Starting - Container lab3-check-a31f544685-api-2 Started -API ready through LB: 7f6a80b69e32, f973eaec4d52 + Container lab3-check-2418557caa-postgres-1 Waiting + Container lab3-check-2418557caa-postgres-1 Healthy + Container lab3-check-2418557caa-migrate-1 Starting + Container lab3-check-2418557caa-migrate-1 Started + Container lab3-check-2418557caa-migrate-1 Waiting + Container lab3-check-2418557caa-postgres-1 Waiting + Container lab3-check-2418557caa-valkey-1 Waiting + Container lab3-check-2418557caa-valkey-1 Healthy + Container lab3-check-2418557caa-postgres-1 Healthy + Container lab3-check-2418557caa-migrate-1 Exited + Container lab3-check-2418557caa-api-2 Starting + Container lab3-check-2418557caa-api-2 Started +API ready through LB: 8e14050120c6, 998bd7b19b89 +Stopped API: 8e14050120c6 container=8e14050120c607ce540e41301256a08292ff7b832d7b32682871ad3c1facf560 +{"label": "stop 8e14050120c6", "duration_seconds": 16.014, "successful_requests": 1452, "successful_writes": 726, "errors": 4, "successful_rps": 90.67, "first_success_after_action_seconds": {"POST": 0.033, "GET": 0.044}, "maximum_success_gap_seconds": {"POST": 2.009, "GET": 2.003}, "action_start_seconds": 1.001, "action_end_seconds": 1.722, "p50_ms": 9.064, "p95_ms": 20.562, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["8e14050120c6", "998bd7b19b89"]} + Container lab3-check-2418557caa-postgres-1 Waiting + Container lab3-check-2418557caa-postgres-1 Healthy + Container lab3-check-2418557caa-migrate-1 Starting + Container lab3-check-2418557caa-migrate-1 Started + Container lab3-check-2418557caa-migrate-1 Waiting + Container lab3-check-2418557caa-postgres-1 Waiting + Container lab3-check-2418557caa-valkey-1 Waiting + Container lab3-check-2418557caa-postgres-1 Healthy + Container lab3-check-2418557caa-valkey-1 Healthy + Container lab3-check-2418557caa-migrate-1 Exited + Container lab3-check-2418557caa-api-1 Starting + Container lab3-check-2418557caa-api-1 Started +API ready through LB: 8e14050120c6, 998bd7b19b89 +Stopped API: 998bd7b19b89 container=998bd7b19b8910cbf3e823a15144b6bdbe42a64e15bb9ae602cdbc30a7e3e190 +{"label": "stop 998bd7b19b89", "duration_seconds": 16.015, "successful_requests": 1504, "successful_writes": 751, "errors": 4, "successful_rps": 93.91, "first_success_after_action_seconds": {"POST": 0.015, "GET": 0.021}, "maximum_success_gap_seconds": {"POST": 1.968, "GET": 1.972}, "action_start_seconds": 1.011, "action_end_seconds": 1.701, "p50_ms": 8.418, "p95_ms": 16.635, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["8e14050120c6", "998bd7b19b89"]} + Container lab3-check-2418557caa-postgres-1 Waiting + Container lab3-check-2418557caa-postgres-1 Healthy + Container lab3-check-2418557caa-migrate-1 Starting + Container lab3-check-2418557caa-migrate-1 Started + Container lab3-check-2418557caa-migrate-1 Waiting + Container lab3-check-2418557caa-postgres-1 Waiting + Container lab3-check-2418557caa-valkey-1 Waiting + Container lab3-check-2418557caa-postgres-1 Healthy + Container lab3-check-2418557caa-valkey-1 Healthy + Container lab3-check-2418557caa-migrate-1 Exited + Container lab3-check-2418557caa-api-2 Starting + Container lab3-check-2418557caa-api-2 Started +API ready through LB: 8e14050120c6, 998bd7b19b89 Local .env and secrets are ready; values are not printed #1 [internal] load local bake definitions #1 reading from stdin 1.03kB done @@ -256,25 +264,25 @@ Local .env and secrets are ready; values are not printed #5 DONE 0.0s #6 [1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 -#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 0.0s done +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 done #6 DONE 0.0s -#7 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#7 [2/8] WORKDIR /app #7 CACHED -#8 [3/8] COPY requirements.txt . +#8 [7/8] COPY migrations/ ./migrations/ #8 CACHED -#9 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#9 [3/8] COPY requirements.txt . #9 CACHED -#10 [6/8] COPY src/ ./src/ +#10 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app #10 CACHED -#11 [2/8] WORKDIR /app +#11 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check #11 CACHED -#12 [7/8] COPY migrations/ ./migrations/ +#12 [6/8] COPY src/ ./src/ #12 CACHED #13 [8/8] COPY research/ ./research/ @@ -282,10 +290,10 @@ Local .env and secrets are ready; values are not printed #14 exporting to image #14 exporting layers done -#14 exporting manifest sha256:06163e45d3676c7cf4f7a1c7bfe108ed3b98559ebb485c7a09d1e87e6f418d17 done -#14 exporting config sha256:3fed7460e9510eef7b2928f8d600d06d5d65329f74d820beac13809476b11398 done -#14 exporting attestation manifest sha256:3a88ce0975ddcbab53c3ecb185da08ef114e818e9db952983e997cd47add67bf done -#14 exporting manifest list sha256:433a8a3293202a92f092574052b474e8373c0d58649cbcf1c7ebc1af17933d3c done +#14 exporting manifest sha256:f5b0ba5ed30baa51240061b64983bd1e0be184515995717d9d42c0f303b28386 done +#14 exporting config sha256:ba8834f54a12be5f335a7d34fccbeb70b0c1b7f1535c33e1a8bf0510bedb4d72 done +#14 exporting attestation manifest sha256:4d6c5d31589d68559a42b790b99b4a58122d4f0ef087bacafbc8c14138d20153 0.0s done +#14 exporting manifest list sha256:fb863cfc900482faa092f947ff3814cc5dc7a270ffadc707e7fd0d1de642873a done #14 naming to docker.io/library/mtusi-messenger:lab3 done #14 unpacking to docker.io/library/mtusi-messenger:lab3 done #14 DONE 0.1s @@ -294,7 +302,7 @@ Local .env and secrets are ready; values are not printed #15 DONE 0.0s mtusi-messenger:lab3 Built Local .env and secrets are ready; values are not printed - Container lab3-check-a31f544685-postgres-1 Running + Container lab3-check-2418557caa-postgres-1 Running #1 [internal] load local bake definitions #1 reading from stdin 576B done #1 DONE 0.0s @@ -315,25 +323,25 @@ Local .env and secrets are ready; values are not printed #5 DONE 0.0s #6 [1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 -#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 done +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 0.0s done #6 DONE 0.0s -#7 [3/8] COPY requirements.txt . +#7 [6/8] COPY src/ ./src/ #7 CACHED -#8 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#8 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app #8 CACHED -#9 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#9 [7/8] COPY migrations/ ./migrations/ #9 CACHED -#10 [6/8] COPY src/ ./src/ +#10 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check #10 CACHED -#11 [7/8] COPY migrations/ ./migrations/ +#11 [2/8] WORKDIR /app #11 CACHED -#12 [2/8] WORKDIR /app +#12 [3/8] COPY requirements.txt . #12 CACHED #13 [8/8] COPY research/ ./research/ @@ -341,63 +349,63 @@ Local .env and secrets are ready; values are not printed #14 exporting to image #14 exporting layers done -#14 exporting manifest sha256:a59568d34e2c5e8b4d23308342601fdaf8c07e78ec3065878679a067c082e952 done -#14 exporting config sha256:b67b805ba1b7dd90f0a3e8444455488fe8c7f4118f14d671501dd63e85b59c97 done -#14 exporting attestation manifest sha256:25ff7934ca5161da9b6c345af61b6a10b452580e4d349b35939d32feb66664ea done -#14 exporting manifest list sha256:3e102cd9a2c1e81e95aed4a0d4de1ac94352370595fced350257813f7a07bef8 done +#14 exporting manifest sha256:b9475efa88de4b71344df73089a66ddd577d05384dce92c62b755b04f74934cc done +#14 exporting config sha256:18b572ee6ee932396009b6354f40403917b55bd3d69920671450baff5304f07f done +#14 exporting attestation manifest sha256:82c4ebc3bd535d871d1e2415f644fa0c987421cbe6f7bd05afd95854da34e4bc done +#14 exporting manifest list sha256:6b3faf75764ac8f93967166e18995765c3c9899adae08c10b0df375a8a871db1 done #14 naming to docker.io/library/mtusi-messenger:lab3 done #14 unpacking to docker.io/library/mtusi-messenger:lab3 done -#14 DONE 0.0s +#14 DONE 0.1s #15 resolving provenance for metadata file #15 DONE 0.0s Migrations: already applied -API ready through LB: 7f6a80b69e32, f973eaec4d52 -Replacing API: 7f6a80b69e32 - Container lab3-check-a31f544685-api-2 Running - Container lab3-check-a31f544685-api-3 Creating - Container lab3-check-a31f544685-api-3 Created - Container lab3-check-a31f544685-api-3 Starting - Container lab3-check-a31f544685-api-3 Started -API ready through LB: ca9c7a52da66, f973eaec4d52 -Replacing API: f973eaec4d52 - Container lab3-check-a31f544685-api-3 Running - Container lab3-check-a31f544685-api-4 Creating - Container lab3-check-a31f544685-api-4 Created - Container lab3-check-a31f544685-api-4 Starting - Container lab3-check-a31f544685-api-4 Started -API ready through LB: 244bf0b910eb, ca9c7a52da66 +API ready through LB: 8e14050120c6, 998bd7b19b89 +Replacing API: 8e14050120c6 + Container lab3-check-2418557caa-api-2 Running + Container lab3-check-2418557caa-api-3 Creating + Container lab3-check-2418557caa-api-3 Created + Container lab3-check-2418557caa-api-3 Starting + Container lab3-check-2418557caa-api-3 Started +API ready through LB: 1dde8575f8f4, 998bd7b19b89 +Replacing API: 998bd7b19b89 + Container lab3-check-2418557caa-api-3 Running + Container lab3-check-2418557caa-api-4 Creating + Container lab3-check-2418557caa-api-4 Created + Container lab3-check-2418557caa-api-4 Starting + Container lab3-check-2418557caa-api-4 Started +API ready through LB: 1dde8575f8f4, 27f47ac686fc PASS: all API containers replaced sequentially; stores and LB kept running -{"label": "sequential API replacement", "duration_seconds": 23.774, "successful_requests": 2131, "successful_writes": 1063, "errors": 11, "successful_rps": 89.64, "first_success_after_action_seconds": {"POST": 0.018, "GET": 0.004}, "maximum_success_gap_seconds": {"POST": 1.059, "GET": 2.034}, "action_start_seconds": 1.005, "action_end_seconds": 19.63, "p50_ms": 7.723, "p95_ms": 13.565, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["244bf0b910eb", "7f6a80b69e32", "ca9c7a52da66", "f973eaec4d52"]} +{"label": "sequential API replacement", "duration_seconds": 22.757, "successful_requests": 2072, "successful_writes": 1037, "errors": 8, "successful_rps": 91.05, "first_success_after_action_seconds": {"POST": 0.019, "GET": 0.025}, "maximum_success_gap_seconds": {"POST": 1.007, "GET": 1.074}, "action_start_seconds": 1.008, "action_end_seconds": 19.74, "p50_ms": 8.068, "p95_ms": 14.446, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["1dde8575f8f4", "27f47ac686fc", "8e14050120c6", "998bd7b19b89"]} PASS: all lab3 grade-4 checks completed - Container lab3-check-a31f544685-lb-1 Stopping - Container lab3-check-a31f544685-lb-1 Stopped - Container lab3-check-a31f544685-lb-1 Removing - Container lab3-check-a31f544685-lb-1 Removed - Container lab3-check-a31f544685-api-3 Stopping - Container lab3-check-a31f544685-api-4 Stopping - Container lab3-check-a31f544685-api-4 Stopped - Container lab3-check-a31f544685-api-4 Removing - Container lab3-check-a31f544685-api-4 Removed - Container lab3-check-a31f544685-api-3 Stopped - Container lab3-check-a31f544685-api-3 Removing - Container lab3-check-a31f544685-api-3 Removed - Container lab3-check-a31f544685-migrate-1 Stopping - Container lab3-check-a31f544685-valkey-1 Stopping - Container lab3-check-a31f544685-migrate-1 Stopped - Container lab3-check-a31f544685-migrate-1 Removing - Container lab3-check-a31f544685-migrate-1 Removed - Container lab3-check-a31f544685-postgres-1 Stopping - Container lab3-check-a31f544685-valkey-1 Stopped - Container lab3-check-a31f544685-valkey-1 Removing - Container lab3-check-a31f544685-valkey-1 Removed - Container lab3-check-a31f544685-postgres-1 Stopped - Container lab3-check-a31f544685-postgres-1 Removing - Container lab3-check-a31f544685-postgres-1 Removed - Volume lab3-check-a31f544685_valkey_data Removing - Network lab3-check-a31f544685_default Removing - Volume lab3-check-a31f544685_postgres_data Removing - Volume lab3-check-a31f544685_valkey_data Removed - Volume lab3-check-a31f544685_postgres_data Removed - Network lab3-check-a31f544685_default Removed -Removed only temporary test containers and volumes: lab3-check-a31f544685 + Container lab3-check-2418557caa-lb-1 Stopping + Container lab3-check-2418557caa-lb-1 Stopped + Container lab3-check-2418557caa-lb-1 Removing + Container lab3-check-2418557caa-lb-1 Removed + Container lab3-check-2418557caa-api-4 Stopping + Container lab3-check-2418557caa-api-3 Stopping + Container lab3-check-2418557caa-api-3 Stopped + Container lab3-check-2418557caa-api-3 Removing + Container lab3-check-2418557caa-api-3 Removed + Container lab3-check-2418557caa-api-4 Stopped + Container lab3-check-2418557caa-api-4 Removing + Container lab3-check-2418557caa-api-4 Removed + Container lab3-check-2418557caa-migrate-1 Stopping + Container lab3-check-2418557caa-valkey-1 Stopping + Container lab3-check-2418557caa-migrate-1 Stopped + Container lab3-check-2418557caa-migrate-1 Removing + Container lab3-check-2418557caa-migrate-1 Removed + Container lab3-check-2418557caa-postgres-1 Stopping + Container lab3-check-2418557caa-valkey-1 Stopped + Container lab3-check-2418557caa-valkey-1 Removing + Container lab3-check-2418557caa-valkey-1 Removed + Container lab3-check-2418557caa-postgres-1 Stopped + Container lab3-check-2418557caa-postgres-1 Removing + Container lab3-check-2418557caa-postgres-1 Removed + Volume lab3-check-2418557caa_valkey_data Removing + Volume lab3-check-2418557caa_postgres_data Removing + Network lab3-check-2418557caa_default Removing + Volume lab3-check-2418557caa_valkey_data Removed + Volume lab3-check-2418557caa_postgres_data Removed + Network lab3-check-2418557caa_default Removed +Removed only temporary test containers and volumes: lab3-check-2418557caa diff --git a/evidence/postgres-pause-before.json b/evidence/postgres-pause-before.json new file mode 100644 index 0000000..0a9e168 --- /dev/null +++ b/evidence/postgres-pause-before.json @@ -0,0 +1,10 @@ +{ + "status": 504, + "elapsed_seconds": 8.008, + "json_error": null, + "x_request_id": null, + "x_instance_id": null, + "date": "2026-11-06", + "commit": "64ea89d04046a0d4eb761c8774d0b33cdb363bb0", + "scenario": "Authenticated history GET immediately after pausing PostgreSQL; existing connections warmed" +} diff --git a/src/db.py b/src/db.py index 8580eca..a4c143b 100644 --- a/src/db.py +++ b/src/db.py @@ -9,8 +9,24 @@ from src.config import secret from src.errors import ApiError +IO_TIMEOUT_SECONDS = 2 + + +class TimeoutConnection(psycopg.Connection): + def wait(self, gen, interval=0.1, timeout=None): + # statement_timeout не работает, пока процесс PostgreSQL приостановлен. + limit = IO_TIMEOUT_SECONDS if timeout is None else min(timeout, IO_TIMEOUT_SECONDS) + try: + return super().wait(gen, interval=interval, timeout=limit) + except psycopg.OperationalError: + # Закрываем сразу: ожидание ROLLBACK/cancel тоже может зависнуть. + self.close() + raise + + pool = ConnectionPool( conninfo=secret("DATABASE_URL"), open=False, + connection_class=TimeoutConnection, min_size=1, max_size=5, max_waiting=20, timeout=2, check=ConnectionPool.check_connection, kwargs={ diff --git a/student-tests/docker_lifecycle.py b/student-tests/docker_lifecycle.py index c5e49e3..2b50c80 100644 --- a/student-tests/docker_lifecycle.py +++ b/student-tests/docker_lifecycle.py @@ -16,6 +16,7 @@ from uuid import uuid4 from helpers import Client, ROOT from client import Failure from resilience import history +from postgres_pause import check_postgres_pause def docker(*args): @@ -212,6 +213,8 @@ time.sleep(30) assert sql("SELECT count(*) FROM messages") == count print("PASS: migrations only in one-shot service; repeated execution preserves data", flush=True) + result["postgres_pause"] = check_postgres_pause(compose, base, lambda: script("wait-ready.sh")) + replay_client = Client(base) replay_actor, _, _, replay_chat = replay_client.bootstrap() replay_key = uuid4().hex diff --git a/student-tests/postgres_pause.py b/student-tests/postgres_pause.py new file mode 100644 index 0000000..c6ee1fc --- /dev/null +++ b/student-tests/postgres_pause.py @@ -0,0 +1,81 @@ +"""Регрессия: замершая БД должна давать API 503 раньше timeout server 8s.""" +import json +import subprocess +import time + +from helpers import Client, ROOT + + +def check_postgres_pause(compose, base_url, wait_ready): + client = Client(base_url, timeout=12) + alice, bob, _, chat = client.bootstrap() + saved = client.message(alice, chat, "До приостановки PostgreSQL")["json"] + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + # Прогреваем соединения обеих реплик: пауза должна затронуть открытые TCP. + for _ in range(8): + client.call("GET", path, actor=bob, schema="MessagePage") + api_id = compose("ps", "-q", "api").split()[0] + compose("pause", "postgres") + try: + started = time.monotonic() + response = client.call("GET", path, 503, actor=bob) + elapsed = time.monotonic() - started + assert elapsed < 6, "API должен завершить ожидание до HAProxy timeout server 8s" + assert response["json"]["error"]["code"] == "unavailable" + # Client проверяет JSON Error, X-Request-Id, X-Instance-Id и совпадение request_id. + report = {"history_status": 503, "history_seconds": round(elapsed, 3), + "error_code": "unavailable", "api_instance": response["headers"]["X-Instance-Id"]} + finally: + compose("unpause", "postgres") + wait_ready() + page = client.call("GET", path, actor=bob, schema="MessagePage")["json"] + assert saved in page["items"], "Подтверждённое сообщение потеряно" + report["same_session_and_message_after_unpause"] = True + + # Отдельно проверяем уже выполняющийся SQL, а не только check_connection пула. + code = """import json,time +from src import db +from src.errors import ApiError +db.pool.open(wait=True,timeout=15) +conn=None +started=None +try: + with db.connection() as conn: + conn.execute("SET application_name='lab3-pause-active-query'") + started=time.monotonic() + conn.execute('SELECT pg_sleep(30)') + raise AssertionError('SQL unexpectedly completed') +except ApiError as error: + print(json.dumps({'status':error.status,'code':error.code,'seconds':round(time.monotonic()-started,3),'connection_closed':conn.closed}),flush=True) +finally: + db.pool.close() +""" + worker = subprocess.Popen(["docker", "exec", api_id, "python", "-u", "-c", code], + cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 5 + while True: + active = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", + "SELECT count(*) FROM pg_stat_activity WHERE application_name='lab3-pause-active-query' AND state='active' AND query='SELECT pg_sleep(30)'") + if active == "1": + break + assert time.monotonic() < deadline, "SQL не начал выполняться" + time.sleep(0.05) + compose("pause", "postgres") + stdout, _ = worker.communicate(timeout=6) + assert worker.returncode == 0, "Проверка активного SQL завершилась с ошибкой" + query = json.loads(stdout) + assert query["status"] == 503 and query["code"] == "unavailable" + assert query["connection_closed"] and query["seconds"] < 6 + report["active_query"] = query + finally: + compose("unpause", "postgres") + if worker.poll() is None: + worker.kill() + worker.communicate(timeout=5) + wait_ready() + assert saved in client.call("GET", path, actor=bob, schema="MessagePage")["json"]["items"] + client.message(alice, chat, "Запись после unpause") + report["read_write_after_recovery"] = True + print("PASS: PostgreSQL pause:", json.dumps(report, ensure_ascii=False), flush=True) + return report