From 5236f44412017da344fd9fda2f57b3227133169c Mon Sep 17 00:00:00 2001 From: Chernikov Date: Sat, 7 Nov 2026 16:18:55 +0300 Subject: [PATCH] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D1=82=D0=B0=D0=B9=D0=BC=D0=B0=D1=83=D1=82=20Pos?= =?UTF-8?q?tgreSQL=20=D0=BD=D0=B0=20=D1=81=D1=82=D0=BE=D1=80=D0=BE=D0=BD?= =?UTF-8?q?=D0=B5=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DEMO.md | 8 + Makefile | 4 + REPORT.md | 40 +++++ evidence/README.md | 6 + evidence/postgres-pause-before.txt | 29 ++++ evidence/postgres-pause.txt | 10 ++ evidence/postgres-timeout-offline.txt | 1 + evidence/postgres-timeout-smoke.txt | 1 + evidence/postgres-timeout-tests.txt | 100 +++++++++++++ scripts/check_postgres_pause.py | 208 ++++++++++++++++++++++++++ scripts/verify.py | 10 +- src/messenger/db.py | 25 +++- 12 files changed, 440 insertions(+), 2 deletions(-) create mode 100644 evidence/postgres-pause-before.txt create mode 100644 evidence/postgres-pause.txt create mode 100644 evidence/postgres-timeout-offline.txt create mode 100644 evidence/postgres-timeout-smoke.txt create mode 100644 evidence/postgres-timeout-tests.txt create mode 100644 scripts/check_postgres_pause.py diff --git a/DEMO.md b/DEMO.md index b7705d3..51b0872 100644 --- a/DEMO.md +++ b/DEMO.md @@ -43,6 +43,14 @@ python -X utf8 scripts/lab4_checks.py --only compensation ## Перезапуск всего стенда +Отдельная регрессия замечания к приостановленной PostgreSQL: + +```sh +python -X utf8 scripts/check_postgres_pause.py --output evidence/postgres-pause.txt +``` + +Скрипт сам выполняет pause/unpause: проверяет прямой 503 с каждой реплики, живость API, замену закрытых соединений и восстановление чтения/записи. Также отдельно приостанавливает БД во время запроса, COMMIT и ROLLBACK. PostgreSQL возобновляется в `finally`; API не перезапускаются. + ```sh # Windows python -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080 diff --git a/Makefile b/Makefile index 77430cb..1936222 100644 --- a/Makefile +++ b/Makefile @@ -49,3 +49,7 @@ race: verify: BASE_URL="$(BASE_URL)" $(PYTHON) scripts/verify.py + +.PHONY: test-postgres-pause +test-postgres-pause: + $(PYTHON) scripts/check_postgres_pause.py --base-url "$(BASE_URL)" diff --git a/REPORT.md b/REPORT.md index ccb250d..030da9e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -62,6 +62,8 @@ Upload требует один Content-Length, Transfer-Encoding не подде У boto3 connect timeout 1 с, read timeout 3 с, одна попытка. Пользовательские ошибки дают 400/413/415, отказ хранилища — 503. Readiness зависит от PostgreSQL, Valkey и S3: все нужны обязательному синхронному пути. +PostgreSQL: `connect_timeout=3` ограничивает подключение, таймаут пула 3 с — получение соединения. Дополнительно `BoundedConnection.wait()` ограничивает каждое сетевое ожидание psycopg **3 секундами на стороне API**, включая проверку соединения пулом, запрос, COMMIT и ROLLBACK. Серверные statement/lock/idle timeout остаются дополнительной защитой и не заменяют клиентский предел: при `docker pause postgres` таймер внутри БД тоже остановлен. На клиентском таймауте соединение закрывается в том же потоке и отбрасывается пулом; ожидания сетевого CANCEL/ROLLBACK или незавершённых фоновых запросов в API нет. Ошибка преобразуется в обычный JSON 503. После unpause пул создаёт рабочие соединения без перезапуска API. + ## Приватность и конфигурация В `secrets/s3_config` две identity, анонимной identity нет. Bootstrap/cleanup имеют административный ключ; API получает только Read, Write, List для `messenger-attachments`. API не может создать bucket или читать другой bucket. В SeaweedFS 3.85 Write допускает также удаление объектов; отдельное ограничение Delete не заявляется. Приложение не предоставляет endpoint удаления. @@ -131,6 +133,43 @@ docker compose --profile tools run --rm cleanup --apply # только ста Dry-run сборщика просмотрел 19 объектов и оставил все 19 как свежие; удалено 0. Реальное удаление объекта старше часа в этом коротком прогоне не проверялось. Лог — `evidence/lab4-checks.txt`. Проверка синтетического INSERT временно создаёт trigger только для `lab4-force-rollback.png` и удаляет его в `finally`. +## Исправление по ревью: приостановленная PostgreSQL + +Замечание к `db.py`: server-side таймеры не ограничивали прямой `/health/ready`, если процесс PostgreSQL приостановлен. На исходном образе сценарий воспроизведён: во втором цикле pause прямой запрос не завершился за клиентские 8 с (`evidence/postgres-pause-before.txt`). + +Команда повторения после сборки исправленного API: + +```sh +python -X utf8 scripts/check_postgres_pause.py --output evidence/postgres-pause.txt +# либо make test-postgres-pause +``` + +Тест выполняет HTTP изнутри каждой API-реплики на `127.0.0.1:8080`, минуя HAProxy; внутренние порты на хост не публикуются. Два цикла pause/unpause, по 12 одновременных запросов на каждую из двух реплик: число запросов превышает размер пула 10. БД всегда возобновляется в `finally`. После каждого unpause теми же токенами создаётся и читается сообщение, без перезапуска API. + +Фактический прогон 07.11.2026 (`evidence/postgres-pause.txt`): + +| Проверка | Результат | +| --- | --- | +| 48 прямых GET /health/ready при paused PostgreSQL | все 503; 3,004–3,085 с, корректные error.code и request_id | +| GET /health/live во время pause | 200; 0,004–0,077 с | +| Возврат готовности после unpause, 2 цикла | 0,681 и 0,886 с; прежняя сессия, запись и чтение работают | +| Pause после получения соединения: запрос | таймаут 3,001 с; соединение закрыто | +| Pause перед COMMIT | таймаут 3,001 с; соединение закрыто | +| Pause перед ROLLBACK | таймаут 3,002 с; соединение закрыто, исходная ошибка сохранена | +| Пул размером 1 после каждого из трёх таймаутов | SELECT после unpause успешен в том же пуле; requests_waiting=0 | + +Три секунды — предел одного ожидания драйвера; ожидание очереди пула имеет собственный предел. Таблица содержит измеренное время HTTP именно этого прогона, а не обещание неизменной задержки на любом оборудовании. Таймаут COMMIT означает неопределённый результат операции, а не доказанный rollback; безопасный повтор отправки сообщения использует тот же Idempotency-Key. + +Дополнительная проверка обычного API после исправления: + +```sh +python -X utf8 scripts/verify.py --only offline smoke tests --output-prefix postgres-timeout- +``` + +Логи: `evidence/postgres-timeout-offline.txt`, `evidence/postgres-timeout-smoke.txt`, `evidence/postgres-timeout-tests.txt`. + +Повторный прогон после исправления: offline PASS, публичный smoke PASS, все 95 собственных тестов PASS за 30,857 с. + ## Ограничения и следующие шаги - Один хост, по одному PostgreSQL, Valkey, S3, HAProxy: отказ узла/диска не решён. @@ -146,3 +185,4 @@ Dry-run сборщика просмотрел 19 объектов и остав - [Boto3 put_object](https://docs.aws.amazon.com/boto3/latest/reference/services/s3/client/put_object.html). - [SeaweedFS S3 credentials](https://github.com/seaweedfs/seaweedfs/wiki/S3-Credentials). - [Compose services](https://docs.docker.com/reference/compose-file/services/). +- [psycopg 3.3.6: Connection.wait(timeout) и закрытие соединения](https://github.com/psycopg/psycopg/blob/3.3.6/psycopg/psycopg/connection.py). diff --git a/evidence/README.md b/evidence/README.md index 282234a..90e6e2c 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -11,6 +11,12 @@ - `persistence.txt`: публичная проверка перезапуска API, PostgreSQL, Valkey, S3 и HAProxy с сохранением volumes. - `stack.json`: фактические времена старта контейнеров после restart, версии, ресурсы, volumes и порты; окружение и значения secrets исключены. +Регрессия ревью 07.11.2026: + +- `postgres-pause-before.txt`: воспроизведение зависания старого API до исправления, клиентский timeout 8 с. +- `postgres-pause.txt`: после исправления — 48 прямых 503 за ~3 с, liveness 200, два восстановления без рестарта и закрытие/замена соединений при таймаутах запроса, COMMIT, ROLLBACK. +- `postgres-timeout-offline.txt`, `postgres-timeout-smoke.txt`, `postgres-timeout-tests.txt`: обычные проверки после изменения драйвера. + Методика и пределы выводов — в `REPORT.md`. Идентификаторы объектов и SHA-256 относятся только к синтетическим данным; tokens и credentials не сохраняются. Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. diff --git a/evidence/postgres-pause-before.txt b/evidence/postgres-pause-before.txt new file mode 100644 index 0000000..424203c --- /dev/null +++ b/evidence/postgres-pause-before.txt @@ -0,0 +1,29 @@ +{"cycle": 1, "api": 1, "status": 503, "requests": 12, "seconds": [3.07, 3.061, 3.061, 3.058, 3.059, 3.012, 3.012, 3.013, 3.014, 3.012, 3.01, 3.011], "live_seconds": 0.01} +{"cycle": 1, "api": 2, "status": 503, "requests": 12, "seconds": [3.092, 3.021, 3.023, 3.019, 3.076, 3.006, 3.07, 3.071, 3.072, 3.007, 3.006, 3.004], "live_seconds": 0.003} +{"cycle": 1, "recovered_seconds": 0.677, "existing_session_and_write": "PASS"} +FAIL: docker compose exec --index 1: ~~~~~^^ + File "/usr/local/lib/python3.14/concurrent/futures/_base.py", line 454, in result + return self.__get_result() + ~~~~~~~~~~~~~~~~~^^ + File "/usr/local/lib/python3.14/concurrent/futures/_base.py", line 396, in __get_result + raise self._exception + File "/usr/local/lib/python3.14/concurrent/futures/thread.py", line 86, in run + result = ctx.run(self.task) + File "/usr/local/lib/python3.14/concurrent/futures/thread.py", line 73, in run + return fn(*args, **kwargs) + File "", line 11, in request + response = c.getresponse() + File "/usr/local/lib/python3.14/http/client.py", line 1478, in getresponse + response.begin() + ~~~~~~~~~~~~~~^^ + File "/usr/local/lib/python3.14/http/client.py", line 343, in begin + version, status, reason = self._read_status() + ~~~~~~~~~~~~~~~~~^^ + File "/usr/local/lib/python3.14/http/client.py", line 304, in _read_status + line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") + ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.14/socket.py", line 729, in readinto + return self._sock.recv_into(b) + ~~~~~~~~~~~~~~~~~~~~^^^ +TimeoutError: timed out + diff --git a/evidence/postgres-pause.txt b/evidence/postgres-pause.txt new file mode 100644 index 0000000..71e1124 --- /dev/null +++ b/evidence/postgres-pause.txt @@ -0,0 +1,10 @@ +{"cycle": 1, "api": 1, "status": 503, "requests": 12, "seconds": [3.065, 3.068, 3.066, 3.065, 3.004, 3.006, 3.005, 3.009, 3.008, 3.008, 3.008, 3.007], "live_seconds": 0.006} +{"cycle": 1, "api": 2, "status": 503, "requests": 12, "seconds": [3.067, 3.013, 3.004, 3.005, 3.009, 3.006, 3.007, 3.008, 3.007, 3.007, 3.006, 3.007], "live_seconds": 0.006} +{"cycle": 1, "recovered_seconds": 0.681, "existing_session_and_write": "PASS"} +{"cycle": 2, "api": 1, "status": 503, "requests": 12, "seconds": [3.071, 3.018, 3.006, 3.062, 3.006, 3.006, 3.007, 3.007, 3.004, 3.004, 3.005, 3.006], "live_seconds": 0.004} +{"cycle": 2, "api": 2, "status": 503, "requests": 12, "seconds": [3.022, 3.085, 3.015, 3.081, 3.015, 3.081, 3.007, 3.079, 3.075, 3.075, 3.076, 3.075], "live_seconds": 0.077} +{"cycle": 2, "recovered_seconds": 0.886, "existing_session_and_write": "PASS"} +{"phase": "query", "timeout_seconds": 3.001, "connection_closed": true, "pool_max_size": 1, "recovered_seconds": 0.002} +{"phase": "commit", "timeout_seconds": 3.001, "connection_closed": true, "pool_max_size": 1, "recovered_seconds": 0.002} +{"phase": "rollback", "timeout_seconds": 3.002, "connection_closed": true, "pool_max_size": 1, "recovered_seconds": 0.002} +PASS: paused PostgreSQL returns bounded direct 503; query/commit/rollback discard connections; same pools recover. diff --git a/evidence/postgres-timeout-offline.txt b/evidence/postgres-timeout-offline.txt new file mode 100644 index 0000000..af45ae2 --- /dev/null +++ b/evidence/postgres-timeout-offline.txt @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/postgres-timeout-smoke.txt b/evidence/postgres-timeout-smoke.txt new file mode 100644 index 0000000..43d3d2c --- /dev/null +++ b/evidence/postgres-timeout-smoke.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/postgres-timeout-tests.txt b/evidence/postgres-timeout-tests.txt new file mode 100644 index 0000000..3e3836c --- /dev/null +++ b/evidence/postgres-timeout-tests.txt @@ -0,0 +1,100 @@ +test_authentication_is_checked_before_body_and_resource (test_access.AuthenticationOrderTest.test_authentication_is_checked_before_body_and_resource) ... ok +test_registration_does_not_need_a_token (test_access.AuthenticationOrderTest.test_registration_does_not_need_a_token) ... ok +test_chat_list_contains_only_own_chats (test_access.ForeignChatTest.test_chat_list_contains_only_own_chats) ... ok +test_every_member_can_read_and_write (test_access.ForeignChatTest.test_every_member_can_read_and_write) ... ok +test_foreign_chat_looks_like_missing_chat (test_access.ForeignChatTest.test_foreign_chat_looks_like_missing_chat) ... ok +test_invalid_chat_id_is_400 (test_access.ForeignChatTest.test_invalid_chat_id_is_400) ... ok +test_outsider_cannot_get_chat (test_access.ForeignChatTest.test_outsider_cannot_get_chat) ... ok +test_outsider_cannot_read_history (test_access.ForeignChatTest.test_outsider_cannot_read_history) ... ok +test_outsider_cannot_write (test_access.ForeignChatTest.test_outsider_cannot_write) ... ok +test_unknown_route_is_404 (test_access.ForeignChatTest.test_unknown_route_is_404) ... ok +test_x_user_id_does_not_open_a_foreign_chat (test_access.ForeignChatTest.test_x_user_id_does_not_open_a_foreign_chat) ... ok +test_attachment_only_and_idempotency (test_attachments.AttachmentsTest.test_attachment_only_and_idempotency) ... ok +test_byte_limit_before_body (test_attachments.AttachmentsTest.test_byte_limit_before_body) ... ok +test_corrupt_and_wrong_mime (test_attachments.AttachmentsTest.test_corrupt_and_wrong_mime) ... ok +test_filename (test_attachments.AttachmentsTest.test_filename) ... ok +test_foreign_chat_upload (test_attachments.AttachmentsTest.test_foreign_chat_upload) ... ok +test_jpeg_original (test_attachments.AttachmentsTest.test_jpeg_original) ... ok +test_original_and_metadata_and_acl (test_attachments.AttachmentsTest.test_original_and_metadata_and_acl) ... ok +test_pixel_limit_before_decode (test_attachments.AttachmentsTest.test_pixel_limit_before_decode) ... ok +test_unknown_foreign_and_duplicate_ids (test_attachments.AttachmentsTest.test_unknown_foreign_and_duplicate_ids) ... ok +test_errors_also_carry_instance_id (test_balancing.BalancingTest.test_errors_also_carry_instance_id) ... ok +test_message_written_on_one_replica_is_read_on_another (test_balancing.BalancingTest.test_message_written_on_one_replica_is_read_on_another) ... ok +test_one_session_is_served_by_several_replicas (test_balancing.BalancingTest.test_one_session_is_served_by_several_replicas) ... ok +test_100_messages_from_10_clients (test_concurrency.ConcurrencyTest.test_100_messages_from_10_clients) ... ok +test_chats_created_in_parallel_are_all_listed (test_concurrency.ConcurrencyTest.test_chats_created_in_parallel_are_all_listed) ... ok +test_same_username_registered_in_parallel_wins_once (test_concurrency.ConcurrencyTest.test_same_username_registered_in_parallel_wins_once) ... ok +test_failed_validation_is_not_remembered (test_idempotency.IdempotencyTest.test_failed_validation_is_not_remembered) ... ok +test_invalid_keys_are_400 (test_idempotency.IdempotencyTest.test_invalid_keys_are_400) ... ok +test_json_formatting_does_not_matter (test_idempotency.IdempotencyTest.test_json_formatting_does_not_matter) ... ok +test_key_format_limits_are_accepted (test_idempotency.IdempotencyTest.test_key_format_limits_are_accepted) ... ok +test_key_scope_is_user_and_chat (test_idempotency.IdempotencyTest.test_key_scope_is_user_and_chat) ... ok +test_missing_key_is_400 (test_idempotency.IdempotencyTest.test_missing_key_is_400) ... ok +test_parallel_requests_with_one_key_create_one_message (test_idempotency.IdempotencyTest.test_parallel_requests_with_one_key_create_one_message) ... ok +test_replay_checks_the_session_again (test_idempotency.IdempotencyTest.test_replay_checks_the_session_again) ... ok +test_replay_returns_the_same_message (test_idempotency.IdempotencyTest.test_replay_returns_the_same_message) ... ok +test_same_key_with_another_body_is_409 (test_idempotency.IdempotencyTest.test_same_key_with_another_body_is_409) ... ok +test_chats_are_paged_in_creation_order (test_pagination.ChatListPaginationTest.test_chats_are_paged_in_creation_order) ... ok +test_cursor_of_another_user_is_400 (test_pagination.ChatListPaginationTest.test_cursor_of_another_user_is_400) ... ok +test_default_limit_is_50 (test_pagination.DefaultLimitTest.test_default_limit_is_50) ... ok +test_empty_history (test_pagination.GrowingHistoryTest.test_empty_history) ... ok +test_messages_sent_during_walk_are_not_lost (test_pagination.GrowingHistoryTest.test_messages_sent_during_walk_are_not_lost) ... ok +test_cursor_from_another_chat_is_400 (test_pagination.MessagePaginationTest.test_cursor_from_another_chat_is_400) ... ok +test_default_and_maximum_limit (test_pagination.MessagePaginationTest.test_default_and_maximum_limit) ... ok +test_invalid_cursor_is_400 (test_pagination.MessagePaginationTest.test_invalid_cursor_is_400) ... ok +test_invalid_limit_is_400 (test_pagination.MessagePaginationTest.test_invalid_limit_is_400) ... ok +test_last_full_page_has_no_cursor (test_pagination.MessagePaginationTest.test_last_full_page_has_no_cursor) ... ok +test_limit_can_change_between_pages (test_pagination.MessagePaginationTest.test_limit_can_change_between_pages) ... ok +test_one_message_per_page (test_pagination.MessagePaginationTest.test_one_message_per_page) ... ok +test_walk_over_several_pages (test_pagination.MessagePaginationTest.test_walk_over_several_pages) ... ok +test_basic_is_rejected_on_business_endpoints (test_sessions.BearerTest.test_basic_is_rejected_on_business_endpoints) ... ok +test_protected_endpoints_need_a_token (test_sessions.BearerTest.test_protected_endpoints_need_a_token) ... ok +test_scheme_name_is_case_insensitive (test_sessions.BearerTest.test_scheme_name_is_case_insensitive) ... ok +test_token_selects_the_user (test_sessions.BearerTest.test_token_selects_the_user) ... ok +test_unknown_or_malformed_token (test_sessions.BearerTest.test_unknown_or_malformed_token) ... ok +test_x_user_id_is_ignored (test_sessions.BearerTest.test_x_user_id_is_ignored) ... ok +test_credentials_in_json_body_are_not_accepted (test_sessions.LoginTest.test_credentials_in_json_body_are_not_accepted) ... ok +test_each_login_gives_a_new_token (test_sessions.LoginTest.test_each_login_gives_a_new_token) ... ok +test_malformed_basic_header (test_sessions.LoginTest.test_malformed_basic_header) ... ok +test_password_with_colons_and_unicode (test_sessions.LoginTest.test_password_with_colons_and_unicode) ... ok +test_registration_never_returns_password (test_sessions.LoginTest.test_registration_never_returns_password) ... ok +test_successful_login (test_sessions.LoginTest.test_successful_login) ... ok +test_unknown_user_gets_the_same_answer (test_sessions.LoginTest.test_unknown_user_gets_the_same_answer) ... ok +test_wrong_password (test_sessions.LoginTest.test_wrong_password) ... ok +test_login_again_after_logout (test_sessions.LogoutTest.test_login_again_after_logout) ... ok +test_logout_revokes_only_the_current_token (test_sessions.LogoutTest.test_logout_revokes_only_the_current_token) ... ok +test_bad_member_ids (test_validation.ChatValidationTest.test_bad_member_ids) ... ok +test_bad_titles (test_validation.ChatValidationTest.test_bad_titles) ... ok +test_caller_must_be_a_member (test_validation.ChatValidationTest.test_caller_must_be_a_member) ... ok +test_duplicate_members (test_validation.ChatValidationTest.test_duplicate_members) ... ok +test_unknown_member_rolls_back_the_whole_chat (test_validation.ChatValidationTest.test_unknown_member_rolls_back_the_whole_chat) ... ok +test_valid_chat_keeps_member_order (test_validation.ChatValidationTest.test_valid_chat_keeps_member_order) ... ok +test_extra_fields (test_validation.InvalidJsonTest.test_extra_fields) ... ok +test_invalid_utf8 (test_validation.InvalidJsonTest.test_invalid_utf8) ... ok +test_json_that_is_not_an_object (test_validation.InvalidJsonTest.test_json_that_is_not_an_object) ... ok +test_lone_surrogate_is_rejected_not_crashed (test_validation.InvalidJsonTest.test_lone_surrogate_is_rejected_not_crashed) ... ok +test_malformed_json_on_message (test_validation.InvalidJsonTest.test_malformed_json_on_message) ... ok +test_malformed_json_on_register (test_validation.InvalidJsonTest.test_malformed_json_on_register) ... ok +test_nul_character_is_rejected_not_crashed (test_validation.InvalidJsonTest.test_nul_character_is_rejected_not_crashed) ... ok +test_wrong_content_type (test_validation.InvalidJsonTest.test_wrong_content_type) ... ok +test_wrong_types (test_validation.InvalidJsonTest.test_wrong_types) ... ok +test_empty_message_is_400 (test_validation.MessageValidationTest.test_empty_message_is_400) ... ok +test_missing_or_wrong_text (test_validation.MessageValidationTest.test_missing_or_wrong_text) ... ok +test_nul_in_text_is_400 (test_validation.MessageValidationTest.test_nul_in_text_is_400) ... ok +test_oversized_body_is_413 (test_validation.MessageValidationTest.test_oversized_body_is_413) ... ok +test_sender_id_in_body_is_rejected (test_validation.MessageValidationTest.test_sender_id_in_body_is_rejected) ... ok +test_text_length_limit (test_validation.MessageValidationTest.test_text_length_limit) ... ok +test_unicode_text_is_kept (test_validation.MessageValidationTest.test_unicode_text_is_kept) ... ok +test_bad_display_names (test_validation.UserValidationTest.test_bad_display_names) ... ok +test_bad_passwords (test_validation.UserValidationTest.test_bad_passwords) ... ok +test_bad_usernames (test_validation.UserValidationTest.test_bad_usernames) ... ok +test_duplicate_username_is_409 (test_validation.UserValidationTest.test_duplicate_username_is_409) ... ok +test_missing_fields (test_validation.UserValidationTest.test_missing_fields) ... ok +test_password_length_limits_are_accepted (test_validation.UserValidationTest.test_password_length_limits_are_accepted) ... ok +test_unicode_display_name_is_kept (test_validation.UserValidationTest.test_unicode_display_name_is_kept) ... ok +test_username_length_limits_are_accepted (test_validation.UserValidationTest.test_username_length_limits_are_accepted) ... ok + +---------------------------------------------------------------------- +Ran 95 tests in 30.857s + +OK diff --git a/scripts/check_postgres_pause.py b/scripts/check_postgres_pause.py new file mode 100644 index 0000000..475ad4d --- /dev/null +++ b/scripts/check_postgres_pause.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Regression: pause PostgreSQL, direct API 503, discard connection and recover. + +No published replica ports: direct HTTP is sent from inside each API container. +PostgreSQL is always unpaused in finally. Uses existing local synthetic stand. +""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import queue +import subprocess +import sys +import threading +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check + +LOG = [] + + +def record(value): + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + LOG.append(text) + print(text, flush=True) + + +def compose(*args, timeout=40): + proc = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, + text=True, encoding='utf-8', errors='replace', timeout=timeout) + if proc.returncode: + raise RuntimeError('docker compose ' + ' '.join(args[:3]) + ': ' + proc.stderr[-1200:]) + return proc.stdout.strip() + + +PROBE = r''' +from concurrent.futures import ThreadPoolExecutor +import http.client, json, sys, time + +expected, count = map(int, sys.argv[1:]) +def request(path, status): + c = http.client.HTTPConnection('127.0.0.1', 8080, timeout=8) + start = time.monotonic() + try: + c.request('GET', path) + response = c.getresponse() + body = json.loads(response.read()) + assert response.status == status, (response.status, body) + assert response.getheader('X-Request-Id') + assert response.getheader('X-Instance-Id') + if status == 503: + assert body['error']['code'] == 'unavailable' + assert body['error']['request_id'] == response.getheader('X-Request-Id') + return round(time.monotonic() - start, 3) + finally: + c.close() + +with ThreadPoolExecutor(max_workers=count) as executor: + futures = [executor.submit(request, '/health/ready', expected) for _ in range(count)] + live = request('/health/live', 200) + elapsed = [future.result() for future in futures] +print(json.dumps(dict(status=expected, requests=count, seconds=elapsed, live_seconds=live))) +''' + + +WORKER = r''' +import json, sys, time +from messenger.config import read_secret +from messenger.db import create_pool, Repository +from messenger.errors import Unavailable + +class SyntheticRollback(Exception): pass +pool = create_pool(read_secret('DATABASE_URL'), 1) +pool.open(wait=True, timeout=10) +repo = Repository(pool) +phase = sys.argv[1] +try: + try: + with repo._transaction() as conn: + conn.execute('SELECT 1') + print('ARMED', flush=True) + assert sys.stdin.readline().strip() == 'go' + started = time.monotonic() + if phase == 'query': + conn.execute('SELECT 1') + elif phase == 'rollback': + raise SyntheticRollback() + # commit: leave the block while PostgreSQL is paused. + except (Unavailable, SyntheticRollback) as error: + elapsed = time.monotonic() - started + assert conn.closed, 'timed-out connection was returned open' + assert elapsed < 8, elapsed + expected = SyntheticRollback if phase == 'rollback' else Unavailable + assert isinstance(error, expected), type(error) + print('TIMED_OUT', flush=True) + else: + raise AssertionError('operation unexpectedly succeeded while PostgreSQL paused') + assert sys.stdin.readline().strip() == 'resume' + recovery = time.monotonic() + while True: + try: + repo.ping() + break + except Unavailable: + if time.monotonic() - recovery >= 15: + raise + time.sleep(0.1) + assert pool.get_stats().get('requests_waiting', 0) == 0 + print(json.dumps(dict(phase=phase, timeout_seconds=round(elapsed, 3), + connection_closed=conn.closed, pool_max_size=1, + recovered_seconds=round(time.monotonic()-recovery, 3))), flush=True) +finally: + pool.close(timeout=1) +''' + + +def line(proc, timeout=15): + result = queue.Queue() + threading.Thread(target=lambda: result.put(proc.stdout.readline()), daemon=True).start() + try: + return result.get(timeout=timeout).strip() + except queue.Empty: + raise AssertionError('database test worker did not finish in time') from None + + +def ready(c): + start = time.monotonic() + while time.monotonic() - start < 30: + try: + c.call('GET', '/health/ready', schema='Health') + return round(time.monotonic() - start, 3) + except Failure: + time.sleep(0.2) + raise AssertionError('API did not recover after PostgreSQL unpause') + + +def readiness(c, ids): + a, b, eve, chat = c.bootstrap() + for index in range(len(ids)): + compose('exec', '--index', str(index + 1), '-T', 'api', 'python', '-c', PROBE, '200', '1') + for cycle in range(2): + try: + compose('pause', 'postgres') + with ThreadPoolExecutor(max_workers=len(ids)) as executor: + futures = [executor.submit(compose, 'exec', '--index', str(index + 1), '-T', 'api', + 'python', '-c', PROBE, '503', '12') for index in range(len(ids))] + for index, future in enumerate(futures): + record(dict(cycle=cycle + 1, api=index + 1, **json.loads(future.result()))) + finally: + compose('unpause', 'postgres') + elapsed = ready(c) + # Same token and a real DB write/read after recovery, no API restart. + message = c.message(a, chat, 'PostgreSQL pause regression ' + str(cycle))['json'] + page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=b, schema='MessagePage')['json'] + check(message in page['items'], 'confirmed write not visible after unpause') + record(dict(cycle=cycle + 1, recovered_seconds=elapsed, existing_session_and_write='PASS')) + + +def operation(phase): + proc = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'api', 'python', '-u', '-c', WORKER, phase], + cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, encoding='utf-8', errors='replace') + try: + check(line(proc) == 'ARMED', 'worker could not open a transaction') + try: + compose('pause', 'postgres') + proc.stdin.write('go\n') + proc.stdin.flush() + check(line(proc) == 'TIMED_OUT', 'operation did not time out and close connection: ' + phase) + finally: + compose('unpause', 'postgres') + out, err = proc.communicate('resume\n', timeout=25) + check(proc.returncode == 0, 'worker failed: ' + err[-1200:]) + record(json.loads(out)) + finally: + if proc.poll() is None: + proc.kill() + proc.communicate(timeout=5) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default='http://127.0.0.1:8080') + parser.add_argument('--output', type=Path) + args = parser.parse_args() + try: + c = Client(args.base_url, timeout=8) + ready(c) + ids = compose('ps', '-q', 'api').splitlines() + check(len(ids) >= 2, 'start at least two API replicas') + readiness(c, ids) + for phase in ('query', 'commit', 'rollback'): + operation(phase) + ready(c) + record('PASS: paused PostgreSQL returns bounded direct 503; query/commit/rollback discard connections; same pools recover.') + return 0 + except Exception as error: + record('FAIL: ' + str(error)) + return 1 + finally: + if args.output: + args.output.write_text('\n'.join(LOG) + '\n', encoding='utf-8') + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/verify.py b/scripts/verify.py index 765e2b9..55efaea 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Run acceptance checks and save real UTF-8 evidence. Requires an already running stand.""" +import argparse import os from pathlib import Path import subprocess @@ -16,14 +17,21 @@ def main(): ('smoke.txt', ['tests/smoke.py', '--base-url', url]), ('tests.txt', ['-m', 'unittest', 'discover', '-s', 'student-tests', '-v']), ('lab4-checks.txt', ['scripts/lab4_checks.py', '--base-url', url]), + ('postgres-pause.txt', ['scripts/check_postgres_pause.py', '--base-url', url]), ('persistence.txt', ['tests/resilience.py', '--mode', 'persistence', '--action-script', hook, '--base-url', url]), ('stack.txt', ['scripts/stack_evidence.py']), ] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--only', nargs='+', choices=[Path(filename).stem for filename, _ in checks]) + parser.add_argument('--output-prefix', default='', help='Keep a separate set of evidence logs') + options = parser.parse_args() for filename, args in checks: + if options.only and Path(filename).stem not in options.only: + continue result = subprocess.run([sys.executable, '-X', 'utf8', *args], cwd=ROOT, env=dict(os.environ, BASE_URL=url), capture_output=True) content = result.stdout.decode('utf-8', errors='replace') + result.stderr.decode('utf-8', errors='replace') - (ROOT / 'evidence' / filename).write_text(content, encoding='utf-8') + (ROOT / 'evidence' / (options.output_prefix + filename)).write_text(content, encoding='utf-8') print(filename + ': ' + ('PASS' if result.returncode == 0 else 'FAIL'), flush=True) if result.returncode: print(content[-6000:], flush=True) diff --git a/src/messenger/db.py b/src/messenger/db.py index 0e0ab03..42c1302 100644 --- a/src/messenger/db.py +++ b/src/messenger/db.py @@ -21,6 +21,28 @@ log = logging.getLogger('messenger') # прерываются, а не держат соединение из пула бесконечно. SESSION_OPTIONS = '-c statement_timeout=5000 -c lock_timeout=3000 -c idle_in_transaction_session_timeout=10000' +# Отдельный клиентский предел: server-side таймеры не работают при docker pause. +DB_IO_TIMEOUT = 3.0 + + +class BoundedConnection(psycopg.Connection): + """Ограничивает сетевое ожидание, включая pool check, COMMIT и ROLLBACK. + + Connection.wait(timeout=...) поддерживается закреплённым psycopg 3.3.6. + После таймаута протокол может быть посреди ответа: такое соединение нельзя + возвращать в пул или ждать его ROLLBACK. Закрываем его в том же потоке; + pool заменит закрытое соединение. Ожидания server-side CANCEL здесь нет. + """ + + def wait(self, gen, interval=0.1, timeout=None): + limit = DB_IO_TIMEOUT if timeout is None else min(timeout, DB_IO_TIMEOUT) + try: + return super().wait(gen, interval=interval, timeout=limit) + except errors._WaitTimeout as error: + self.close() + raise psycopg.OperationalError('PostgreSQL client I/O timeout') from error + + MEMBER_IDS = 'array(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.position) AS member_ids' @@ -39,6 +61,7 @@ class IdempotencyConflict(Exception): def create_pool(database_url, max_size): return ConnectionPool( database_url, + connection_class=BoundedConnection, min_size=1, max_size=max_size, # не больше max_size соединений с БД на процесс timeout=3, # ждать свободное соединение не дольше 3 с, затем 503 @@ -101,7 +124,7 @@ class Repository: with self._pool.connection() as conn: yield conn except psycopg.OperationalError as error: - # Нет соединения, пул исчерпан, истёк statement_timeout/lock_timeout — всё это 503. + # Включая клиентский I/O timeout, когда процесс PostgreSQL приостановлен. raise Unavailable('postgres') from error def ping(self):