diff --git a/evidence/grade3-faults.txt b/evidence/grade3-faults.txt new file mode 100644 index 00000000..077c35c2 --- /dev/null +++ b/evidence/grade3-faults.txt @@ -0,0 +1,67 @@ +date: 2026-11-05T18:46:25.070058+00:00 commit: 7fc911c332aefe33285ea9e7811c37abd9bb20a6 +environment: COMPOSE_PROJECT_NAME=mtusi2026lab3-grade3-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 NO_PROXY=localhost,127.0.0.1,::1 +$ make test-faults +BASE_URL="http://127.0.0.1:18083" .venv/bin/pytest -m stack -v -s student-tests/stack +============================= test session starts ============================== +platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /.venv/bin/python +cachedir: .pytest_cache +rootdir: +configfile: pyproject.toml +plugins: asyncio-1.4.0, anyio-4.15.1 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function +collecting ... collected 12 items + +student-tests/stack/test_faults.py::test_registration_race_yields_exactly_one_user гонка 50 регистраций: 201×1, 409×49 +PASSED +student-tests/stack/test_faults.py::test_chat_creation_rolls_back_on_unknown_member чат с несуществующим участником: 400 invalid_request +PASSED +student-tests/stack/test_faults.py::test_wrong_password_and_unknown_user_get_same_basic_challenge неверный пароль: 401 за 0.013 с +неизвестный пользователь: 401 за 0.012 с +PASSED +student-tests/stack/test_faults.py::test_logout_then_token_is_dead logout: 204, затем 401, повтор 401 +PASSED +student-tests/stack/test_faults.py::test_valkey_serves_repeated_checks 5 проверок токена: keyspace_hits 4 → 9 +PASSED +student-tests/stack/test_faults.py::test_valkey_stopped_is_bypassed_but_logout_waits valkey остановлен, действующий токен: 200 за 0.005 с +valkey остановлен, неизвестный токен: 401 за 0.003 с +valkey остановлен: ready 200, вход 201, выход 503 +valkey снова запущен: повторный выход 204 +PASSED +student-tests/stack/test_faults.py::test_valkey_restart_keeps_cache_and_revocations рестарт valkey: ключей 19 → 19 +рестарт valkey: живой 200, отозванный 401 +PASSED +student-tests/stack/test_faults.py::test_valkey_flushall_loses_nothing FLUSHALL: живой 200, отозванный 401 +PASSED +student-tests/stack/test_faults.py::test_postgres_stopped_gives_bounded_503_and_recovers postgres остановлен, ready: 503 за 0.003 с +postgres остановлен, me: 503 за 0.002 с +postgres остановлен, stranger: 503 за 0.003 с +postgres остановлен, login: 503 за 0.003 с +postgres остановлен, register: 503 за 0.014 с +live внутри реплики 200; LB исключил все реплики ещё через 1.14 с +live через LB без реплик: 503 за 0.000 с +postgres запущен: готовность через 1.5 с, прежний токен 200 +PASSED +student-tests/stack/test_faults.py::test_api_restart_keeps_session_without_extending_it FAILED +student-tests/stack/test_faults.py::test_full_restart_keeps_session_until_logout restart.sh: готовность через 0.0 с, прежний токен 200, выход 204, затем 401 +PASSED +student-tests/stack/test_faults.py::test_session_expires_after_restart_at_original_time срок сессии сокращён до 20 с, рестарт api: готовность через 1.5 с, до срока 200, после срока 401 +PASSED + +=================================== FAILURES =================================== +_____________ test_api_restart_keeps_session_without_extending_it ______________ + +http = + + async def test_api_restart_keeps_session_without_extending_it(http: AsyncClient) -> None: + body = new_user("restart") +> assert (await http.post("/api/v1/users", json=body)).status_code == 201 +E assert 503 == 201 +E + where 503 = .status_code + +student-tests/stack/test_faults.py:219: AssertionError +=========================== short test summary info ============================ +FAILED student-tests/stack/test_faults.py::test_api_restart_keeps_session_without_extending_it +======================== 1 failed, 11 passed in 43.22s ========================= +make: *** [test-faults] Error 1 + +exit_code: 2 elapsed_seconds: 43.788 diff --git a/evidence/grade3-pool-regression-after.txt b/evidence/grade3-pool-regression-after.txt new file mode 100644 index 00000000..0fe32127 --- /dev/null +++ b/evidence/grade3-pool-regression-after.txt @@ -0,0 +1,9 @@ +date: 2026-11-05T18:52:46.041538+00:00 commit: 18de59574016564ce4eb69da9f4363441ba33f7d +working_tree: pool_pre_ping and test changes before implementation commit +environment: COMPOSE_PROJECT_NAME=mtusi2026lab3-grade3-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 NO_PROXY=localhost,127.0.0.1,::1 +$ make student-test 'PYTEST_ARGS=-q -k "closed_idle_connection or sigterm_leaves"' +.venv/bin/pytest -q -k "closed_idle_connection or sigterm_leaves" +.. [100%] +2 passed, 281 deselected in 4.28s + +exit_code: 0 elapsed_seconds: 4.753 diff --git a/evidence/grade3-pool-regression-before.txt b/evidence/grade3-pool-regression-before.txt new file mode 100644 index 00000000..94c4bc0a --- /dev/null +++ b/evidence/grade3-pool-regression-before.txt @@ -0,0 +1,29 @@ +date: 2026-11-05T18:52:07.642995+00:00 commit: 18de59574016564ce4eb69da9f4363441ba33f7d +working_tree: added regression test in student-tests/api/test_pool.py; application code unchanged +environment: COMPOSE_PROJECT_NAME=mtusi2026lab3-grade3-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 NO_PROXY=localhost,127.0.0.1,::1 +$ make student-test 'PYTEST_ARGS=-q -k closed_idle_connection' +.venv/bin/pytest -q -k closed_idle_connection +F [100%] +=================================== FAILURES =================================== +_________ test_closed_idle_connection_is_replaced_before_registration __________ + + async def test_closed_idle_connection_is_replaced_before_registration() -> None: + async with running_app(db_pool_max_size=1) as http: + database = await resolve(http, Database) + async with database.session() as session: + pid = await session.scalar(select(func.pg_backend_pid())) + async with sql_connection() as connection: + assert await connection.scalar(select(func.pg_terminate_backend(pid, 1000))) + body = new_user("reconnect") + response = await http.post("/api/v1/users", json=body) +> assert response.status_code == 201 +E assert 503 == 201 +E + where 503 = .status_code + +student-tests/api/test_pool.py:78: AssertionError +=========================== short test summary info ============================ +FAILED student-tests/api/test_pool.py::test_closed_idle_connection_is_replaced_before_registration +1 failed, 282 deselected in 1.84s +make: *** [student-test] Error 1 + +exit_code: 2 elapsed_seconds: 2.461 diff --git a/evidence/grade3-student-tests-interrupted.txt b/evidence/grade3-student-tests-interrupted.txt new file mode 100644 index 00000000..5ed40c9a --- /dev/null +++ b/evidence/grade3-student-tests-interrupted.txt @@ -0,0 +1,69 @@ +date: 2026-11-05T18:47:00.548192+00:00 commit: 7fc911c332aefe33285ea9e7811c37abd9bb20a6 +environment: COMPOSE_PROJECT_NAME=mtusi2026lab3-grade3-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 NO_PROXY=localhost,127.0.0.1,::1 +$ make student-test PYTEST_ARGS=-vv +.venv/bin/pytest -vv +============================= test session starts ============================== +platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /.venv/bin/python +cachedir: .pytest_cache +rootdir: +configfile: pyproject.toml +testpaths: student-tests +plugins: asyncio-1.4.0, anyio-4.15.1 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function +collecting ... collected 282 items / 12 deselected / 270 selected + +student-tests/api/test_access.py::test_outsider_gets_404_on_chat PASSED [ 0%] +student-tests/api/test_access.py::test_unknown_chat_is_indistinguishable_from_foreign PASSED [ 0%] +student-tests/api/test_access.py::test_outsider_cannot_write PASSED [ 1%] +student-tests/api/test_access.py::test_outsider_cannot_read_history PASSED [ 1%] +student-tests/api/test_access.py::test_access_checked_before_body PASSED [ 1%] +student-tests/api/test_access.py::test_chat_list_hides_foreign_chats PASSED [ 2%] +student-tests/api/test_access.py::test_403_is_never_used PASSED [ 2%] +student-tests/api/test_access.py::test_request_shape_is_checked_before_access PASSED [ 2%] +student-tests/api/test_access.py::test_outsider_does_not_wait_for_members_lock PASSED [ 3%] +student-tests/api/test_bounded_contexts.py::test_identity_does_not_know_about_messaging PASSED [ 3%] +student-tests/api/test_bounded_contexts.py::test_domains_do_not_import_infrastructure PASSED [ 4%] +student-tests/api/test_bounded_contexts.py::test_core_layers_do_not_import_pydantic PASSED [ 4%] +student-tests/api/test_bounded_contexts.py::test_shared_knows_nothing_about_zones PASSED [ 4%] +student-tests/api/test_bounded_contexts.py::test_shared_core_is_free_of_transport PASSED [ 5%] +student-tests/api/test_bounded_contexts.py::test_shared_core_holds_only_platform PASSED [ 5%] +student-tests/api/test_bounded_contexts.py::test_shared_entrypoints_hold_only_transport PASSED [ 5%] +student-tests/api/test_bounded_contexts.py::test_every_zone_owns_its_errors_events_and_commands PASSED [ 6%] +student-tests/api/test_bounded_contexts.py::test_every_zone_owns_its_http_layer PASSED [ 6%] +student-tests/api/test_bounded_contexts.py::test_only_main_knows_both_zones PASSED [ 7%] +student-tests/api/test_bounded_contexts.py::test_application_has_no_separate_entrypoints_root PASSED [ 7%] +student-tests/api/test_bounded_contexts.py::test_shared_source_mentions_no_domain_vocabulary PASSED [ 7%] +student-tests/api/test_bounded_contexts.py::test_shared_declares_no_concrete_commands_or_events PASSED [ 8%] +student-tests/api/test_bounded_contexts.py::test_shared_has_no_test_doubles PASSED [ 8%] +student-tests/api/test_bounded_contexts.py::test_zone_errors_do_not_leak_transport_codes PASSED [ 8%] +student-tests/api/test_bounded_contexts.py::test_sql_of_each_zone_reads_only_its_tables PASSED [ 9%] +student-tests/api/test_bounded_contexts.py::test_messaging_does_not_import_identity_at_all PASSED [ 9%] +student-tests/api/test_bounded_contexts.py::test_sqlalchemy_stays_in_infrastructure_and_asyncpg_is_only_a_driver PASSED [ 10%] +student-tests/api/test_cache.py::test_second_check_is_served_from_cache PASSED [ 10%] +student-tests/api/test_cache.py::test_logout_writes_tombstone_before_database PASSED [ 10%] +student-tests/api/test_cache.py::test_logout_rejected_by_database_changes_nothing PASSED [ 11%] +student-tests/api/test_cache.py::test_fill_racing_with_logout_does_not_revive_token PASSED [ 11%] +student-tests/api/test_cache.py::test_flushall_loses_nothing PASSED [ 11%] +student-tests/api/test_cache.py::test_unreachable_cache_is_bypassed_without_anonymous_access PASSED [ 12%] +student-tests/api/test_concurrency.py::test_hundred_messages_from_ten_clients PASSED [ 12%] +student-tests/api/test_concurrency.py::test_history_follows_the_clock_but_never_moves_back PASSED [ 12%] +student-tests/api/test_concurrency.py::test_parallel_registration_keeps_username_unique PASSED [ 13%] +student-tests/api/test_concurrency.py::test_parallel_chat_creation_is_atomic PASSED [ 13%] +student-tests/api/test_consistency.py::test_registration_is_visible_to_chats_immediately PASSED [ 14%] +student-tests/api/test_consistency.py::test_parallel_registrations_are_all_usable_as_members PASSED [ 14%] +student-tests/api/test_consistency.py::test_rejected_registration_leaves_no_trace PASSED [ 14%] +student-tests/api/test_consistency.py::test_failed_chat_creation_rolls_back_both_tables PASSED [ 15%] +student-tests/api/test_consistency.py::test_rejected_message_does_not_enter_history PASSED [ 15%] +student-tests/api/test_consistency.py::test_confirmed_write_is_visible_to_next_read PASSED [ 15%] +student-tests/api/test_consistency.py::test_parallel_writes_across_zones_keep_counts_consistent PASSED [ 16%] +student-tests/api/test_consistency.py::test_chat_membership_survives_parallel_reads_and_writes PASSED [ 16%] +student-tests/api/test_drain.py::test_draining_replica_is_not_ready_but_alive PASSED [ 17%] +student-tests/api/test_drain.py::test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! KeyboardInterrupt !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +/student-tests/api/test_drain.py:101: KeyboardInterrupt +(to show a full traceback on KeyboardInterrupt use --full-trace) +================ 46 passed, 12 deselected in 275.91s (0:04:35) ================= +make: *** [student-test] Error 2 + +exit_code: 2 elapsed_seconds: 276.378 diff --git a/evidence/grade3-student-tests-startup.txt.gz b/evidence/grade3-student-tests-startup.txt.gz new file mode 100644 index 00000000..3855cc9b Binary files /dev/null and b/evidence/grade3-student-tests-startup.txt.gz differ diff --git a/openspec/changes/fix-lab3-postgres-pool-recovery/design.md b/openspec/changes/fix-lab3-postgres-pool-recovery/design.md index 99e55e24..c7e2200f 100644 --- a/openspec/changes/fix-lab3-postgres-pool-recovery/design.md +++ b/openspec/changes/fix-lab3-postgres-pool-recovery/design.md @@ -20,6 +20,12 @@ - Дополнительное обращение к PostgreSQL → стоимость измеряется в следующем этапе трека, старые RPS не объявляются результатом SQLAlchemy. - Разрыв после успешной проверки всё ещё возможен → существующие ограниченные `503`, сообщение повторяется клиентом с тем же ключом. +## Изоляция собственного теста SIGTERM + +Полный набор при работающем LB завис на `test_drain.py`: встроенный gunicorn тоже пытался слушать 18083, а тест принимал readiness LB за готовность дочернего процесса. Прогон прерван после 46 успешных тестов, вывод сохранён в `evidence/grade3-student-tests-interrupted.txt`. Тест выбирает свободный порт 23000–23999 через существующую fixture-утилиту и ограничивает ожидание закрытия listener. Поведение приложения не меняется. + +Следующий полный прогон выявил стартовую гонку тестовой БД: SQLAlchemy пробрасывает `CannotConnectNowError` с SQLSTATE `57P03` при открытии соединения до готовности PostgreSQL. Fixture миграций использует существующий классификатор `unavailable`, который учитывает SQLSTATE, и повторяет только временные отказы до прежнего дедлайна 60 с. Ошибки схемы не скрываются. Исходный вывод (70 passed / 201 errors) сохранён в `evidence/grade3-student-tests-startup.txt.gz`. + ## Migration Plan Миграция схемы не нужна. Собрать текущий API, переснять публичные smoke/persistence/failover обеих реплик, гонки ключа и потерянный ответ, `make test-faults`, собственные тесты. Сохранить исходное падение отдельно. Откат — предыдущий образ на той же схеме и volumes, с известной регрессией пула. diff --git a/openspec/changes/fix-lab3-postgres-pool-recovery/proposal.md b/openspec/changes/fix-lab3-postgres-pool-recovery/proposal.md index ddfcf758..42937fa0 100644 --- a/openspec/changes/fix-lab3-postgres-pool-recovery/proposal.md +++ b/openspec/changes/fix-lab3-postgres-pool-recovery/proposal.md @@ -6,6 +6,7 @@ - Проверять соединение пула перед началом операции и заменять закрытое; не повторять уже начатые транзакции и неидемпотентные POST. - Добавить регрессионную проверку разрыва простаивающего соединения и усилить проверку записей после возвращения PostgreSQL через LB. +- Исправить выявленные проблемы запуска собственных тестов: коллизию порта gunicorn с LB и стартовую гонку тестовой БД. - Переснять уровень «3» и обновить строки «Перенос lab2: отказы PostgreSQL» и «3.5» в `REPORT.md`, сохранив неуспешный прогон. ## Capabilities diff --git a/openspec/changes/fix-lab3-postgres-pool-recovery/tasks.md b/openspec/changes/fix-lab3-postgres-pool-recovery/tasks.md index df6b5acf..9dd0833f 100644 --- a/openspec/changes/fix-lab3-postgres-pool-recovery/tasks.md +++ b/openspec/changes/fix-lab3-postgres-pool-recovery/tasks.md @@ -1,7 +1,9 @@ ## 1. Регрессия и исправление платформы -- [ ] 1.1 Воспроизвести первый `503` после закрытия простаивающего соединения отдельным тестом `test_pool.py`, сохранить исходное падение -- [ ] 1.2 Включить проверку соединения до выдачи из пула; целевой тест и `make student-test`, `make lint`, `make check` проходят +- [x] 1.1 Воспроизвести первый `503` после закрытия простаивающего соединения отдельным тестом `test_pool.py`, сохранить исходное падение +- [x] 1.2 Включить проверку соединения до выдачи из пула; целевой тест и `make student-test`, `make lint`, `make check` проходят +- [ ] 1.3 Изолировать порт gunicorn в `test_drain.py` от LB и ограничить ожидание закрытия listener; тест и полный набор проходят при работающем стенде +- [x] 1.4 Дождаться готовности тестового PostgreSQL при SQLSTATE 57P03 через существующий классификатор временных отказов; полный набор проходит ## 2. Реальный стенд уровня «3» diff --git a/src/messenger/shared/infrastructure/postgres.py b/src/messenger/shared/infrastructure/postgres.py index 5b3531e2..b2d5f550 100644 --- a/src/messenger/shared/infrastructure/postgres.py +++ b/src/messenger/shared/infrastructure/postgres.py @@ -56,6 +56,7 @@ class Database: max_overflow=0, pool_timeout=acquire_timeout, pool_recycle=60, + pool_pre_ping=True, hide_parameters=True, connect_args={ "timeout": connect_timeout, diff --git a/student-tests/api/test_drain.py b/student-tests/api/test_drain.py index fe5fccc1..58462672 100644 --- a/student-tests/api/test_drain.py +++ b/student-tests/api/test_drain.py @@ -9,7 +9,7 @@ from pathlib import Path import httpx -from conftest import DATABASE, new_user +from conftest import DATABASE, free_port, new_user from messenger.shared.infrastructure.readiness import DRAINING DRAIN = 1.0 @@ -67,7 +67,7 @@ def begin_registration(port: int) -> tuple[socket.socket, bytes]: def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests(tmp_path: Path) -> None: - port = 18083 + port = free_port() settings = { "HOST": "127.0.0.1", "PORT": str(port), @@ -98,6 +98,7 @@ def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests(tmp_path time.sleep(0.3) during = status(port, "/health/ready"), status(port, "/health/live") while status(port, "/health/live") == 200: + assert time.monotonic() - sent < SHUTDOWN + 2, log_path.read_text() time.sleep(0.05) closed = time.monotonic() - sent pending.sendall(rest) diff --git a/student-tests/api/test_pool.py b/student-tests/api/test_pool.py index 6f64c661..3ccb5881 100644 --- a/student-tests/api/test_pool.py +++ b/student-tests/api/test_pool.py @@ -7,11 +7,21 @@ import pytest from sqlalchemy import func, select from sqlalchemy.exc import DBAPIError, IntegrityError -from conftest import actor, keyed, make_chat, register, resolve, running_app +from conftest import ( + actor, + keyed, + make_chat, + new_user, + register, + resolve, + running_app, + sql_connection, +) from messenger.identity.domain.models import Session from messenger.identity.infrastructure.postgres import ( PostgresIdentityUnitOfWork, PostgresSessionDirectory, + UserRow, ) from messenger.messaging.domain.models import Message from messenger.messaging.domain.values import MessageText @@ -65,6 +75,28 @@ async def test_idle_session_does_not_acquire_connection() -> None: await database.ping() +async def test_closed_idle_connection_is_replaced_before_registration() -> None: + async with running_app(db_pool_max_size=1) as http: + database = await resolve(http, Database) + async with database.session() as session: + pid = await session.scalar(select(func.pg_backend_pid())) + async with sql_connection() as connection: + assert await connection.scalar(select(func.pg_terminate_backend(pid, 1000))) + body = new_user("reconnect") + response = await http.post("/api/v1/users", json=body) + assert response.status_code == 201 + assert response.headers["X-Instance-Id"] and response.headers["X-Request-Id"] + async with database.session() as session: + assert ( + await session.scalar( + select(func.count()) + .select_from(UserRow) + .where(UserRow.username == body["username"]) + ) + == 1 + ) + + async def test_readiness_timeout_releases_waiter() -> None: async with running_app(db_pool_max_size=1, db_acquire_timeout_seconds=2) as http: database = await resolve(http, Database) diff --git a/student-tests/conftest.py b/student-tests/conftest.py index 2c6f659a..0d6f7171 100644 --- a/student-tests/conftest.py +++ b/student-tests/conftest.py @@ -16,7 +16,6 @@ from alembic.config import Config from fastapi import FastAPI from httpx import ASGITransport, AsyncClient from sqlalchemy import make_url -from sqlalchemy.exc import DBAPIError from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine from sqlalchemy.pool import NullPool @@ -25,6 +24,7 @@ from messenger.main import create_app from messenger.messaging.config import MessagingSettings from messenger.shared.config import PlatformSettings from messenger.shared.entrypoints.config import TransportSettings +from messenger.shared.infrastructure.postgres import unavailable ROOT = Path(__file__).resolve().parents[1] POSTGRES_IMAGE = re.compile(r"image: (postgres:\S+)") @@ -57,8 +57,8 @@ async def migrated(dsn: str) -> None: try: await upgrade(dsn) return - except OSError, DBAPIError: - if time.monotonic() > deadline: + except Exception as error: + if not unavailable(error) or time.monotonic() > deadline: raise await asyncio.sleep(0.5) diff --git a/student-tests/stack/test_faults.py b/student-tests/stack/test_faults.py index 6f57a9c2..5a67c497 100644 --- a/student-tests/stack/test_faults.py +++ b/student-tests/stack/test_faults.py @@ -211,6 +211,13 @@ async def test_postgres_stopped_gives_bounded_503_and_recovers(http: AsyncClient me = await http.get("/api/v1/users/me", headers=bearer(token)) print(f"postgres запущен: готовность через {recovered:.1f} с, прежний токен {me.status_code}") assert me.json() == user + instances = set() + for _ in range(40): + response = await http.post("/api/v1/users", json=new_user("recovered")) + assert response.status_code == 201 + instances.add(response.headers["X-Instance-Id"]) + print(f"postgres запущен: 40 регистраций 201 через {len(instances)} реплики без рестарта API") + assert len(instances) >= 2 assert started_at("api") == api_started