From 0a1f48bd73e5780c732356880c23fdb476bcff98 Mon Sep 17 00:00:00 2001 From: Artyom Date: Thu, 5 Nov 2026 21:55:14 +0300 Subject: [PATCH] Recover stale PostgreSQL connections before starting requests --- evidence/grade3-faults.txt | 67 +++++++++++++++++ evidence/grade3-pool-regression-after.txt | 9 +++ evidence/grade3-pool-regression-before.txt | 29 ++++++++ evidence/grade3-student-tests-interrupted.txt | 69 ++++++++++++++++++ evidence/grade3-student-tests-startup.txt.gz | Bin 0 -> 37486 bytes .../fix-lab3-postgres-pool-recovery/design.md | 6 ++ .../proposal.md | 1 + .../fix-lab3-postgres-pool-recovery/tasks.md | 6 +- .../shared/infrastructure/postgres.py | 1 + student-tests/api/test_drain.py | 5 +- student-tests/api/test_pool.py | 34 ++++++++- student-tests/conftest.py | 6 +- student-tests/stack/test_faults.py | 7 ++ 13 files changed, 232 insertions(+), 8 deletions(-) create mode 100644 evidence/grade3-faults.txt create mode 100644 evidence/grade3-pool-regression-after.txt create mode 100644 evidence/grade3-pool-regression-before.txt create mode 100644 evidence/grade3-student-tests-interrupted.txt create mode 100644 evidence/grade3-student-tests-startup.txt.gz diff --git a/evidence/grade3-faults.txt b/evidence/grade3-faults.txt new file mode 100644 index 00000000..077c35c2 --- /dev/null +++ b/evidence/grade3-faults.txt @@ -0,0 +1,67 @@ +date: 2026-11-05T18:46:25.070058+00:00 commit: 7fc911c332aefe33285ea9e7811c37abd9bb20a6 +environment: COMPOSE_PROJECT_NAME=mtusi2026lab3-grade3-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 NO_PROXY=localhost,127.0.0.1,::1 +$ make test-faults +BASE_URL="http://127.0.0.1:18083" .venv/bin/pytest -m stack -v -s student-tests/stack +============================= test session starts ============================== +platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /.venv/bin/python +cachedir: .pytest_cache +rootdir: +configfile: pyproject.toml +plugins: asyncio-1.4.0, anyio-4.15.1 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function +collecting ... collected 12 items + +student-tests/stack/test_faults.py::test_registration_race_yields_exactly_one_user гонка 50 регистраций: 201×1, 409×49 +PASSED +student-tests/stack/test_faults.py::test_chat_creation_rolls_back_on_unknown_member чат с несуществующим участником: 400 invalid_request +PASSED +student-tests/stack/test_faults.py::test_wrong_password_and_unknown_user_get_same_basic_challenge неверный пароль: 401 за 0.013 с +неизвестный пользователь: 401 за 0.012 с +PASSED +student-tests/stack/test_faults.py::test_logout_then_token_is_dead logout: 204, затем 401, повтор 401 +PASSED +student-tests/stack/test_faults.py::test_valkey_serves_repeated_checks 5 проверок токена: keyspace_hits 4 → 9 +PASSED +student-tests/stack/test_faults.py::test_valkey_stopped_is_bypassed_but_logout_waits valkey остановлен, действующий токен: 200 за 0.005 с +valkey остановлен, неизвестный токен: 401 за 0.003 с +valkey остановлен: ready 200, вход 201, выход 503 +valkey снова запущен: повторный выход 204 +PASSED +student-tests/stack/test_faults.py::test_valkey_restart_keeps_cache_and_revocations рестарт valkey: ключей 19 → 19 +рестарт valkey: живой 200, отозванный 401 +PASSED +student-tests/stack/test_faults.py::test_valkey_flushall_loses_nothing FLUSHALL: живой 200, отозванный 401 +PASSED +student-tests/stack/test_faults.py::test_postgres_stopped_gives_bounded_503_and_recovers postgres остановлен, ready: 503 за 0.003 с +postgres остановлен, me: 503 за 0.002 с +postgres остановлен, stranger: 503 за 0.003 с +postgres остановлен, login: 503 за 0.003 с +postgres остановлен, register: 503 за 0.014 с +live внутри реплики 200; LB исключил все реплики ещё через 1.14 с +live через LB без реплик: 503 за 0.000 с +postgres запущен: готовность через 1.5 с, прежний токен 200 +PASSED +student-tests/stack/test_faults.py::test_api_restart_keeps_session_without_extending_it FAILED +student-tests/stack/test_faults.py::test_full_restart_keeps_session_until_logout restart.sh: готовность через 0.0 с, прежний токен 200, выход 204, затем 401 +PASSED +student-tests/stack/test_faults.py::test_session_expires_after_restart_at_original_time срок сессии сокращён до 20 с, рестарт api: готовность через 1.5 с, до срока 200, после срока 401 +PASSED + +=================================== FAILURES =================================== +_____________ test_api_restart_keeps_session_without_extending_it ______________ + +http = + + async def test_api_restart_keeps_session_without_extending_it(http: AsyncClient) -> None: + body = new_user("restart") +> assert (await http.post("/api/v1/users", json=body)).status_code == 201 +E assert 503 == 201 +E + where 503 = .status_code + +student-tests/stack/test_faults.py:219: AssertionError +=========================== short test summary info ============================ +FAILED student-tests/stack/test_faults.py::test_api_restart_keeps_session_without_extending_it +======================== 1 failed, 11 passed in 43.22s ========================= +make: *** [test-faults] Error 1 + +exit_code: 2 elapsed_seconds: 43.788 diff --git a/evidence/grade3-pool-regression-after.txt b/evidence/grade3-pool-regression-after.txt new file mode 100644 index 00000000..0fe32127 --- /dev/null +++ b/evidence/grade3-pool-regression-after.txt @@ -0,0 +1,9 @@ +date: 2026-11-05T18:52:46.041538+00:00 commit: 18de59574016564ce4eb69da9f4363441ba33f7d +working_tree: pool_pre_ping and test changes before implementation commit +environment: COMPOSE_PROJECT_NAME=mtusi2026lab3-grade3-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 NO_PROXY=localhost,127.0.0.1,::1 +$ make student-test 'PYTEST_ARGS=-q -k "closed_idle_connection or sigterm_leaves"' +.venv/bin/pytest -q -k "closed_idle_connection or sigterm_leaves" +.. [100%] +2 passed, 281 deselected in 4.28s + +exit_code: 0 elapsed_seconds: 4.753 diff --git a/evidence/grade3-pool-regression-before.txt b/evidence/grade3-pool-regression-before.txt new file mode 100644 index 00000000..94c4bc0a --- /dev/null +++ b/evidence/grade3-pool-regression-before.txt @@ -0,0 +1,29 @@ +date: 2026-11-05T18:52:07.642995+00:00 commit: 18de59574016564ce4eb69da9f4363441ba33f7d +working_tree: added regression test in student-tests/api/test_pool.py; application code unchanged +environment: COMPOSE_PROJECT_NAME=mtusi2026lab3-grade3-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 NO_PROXY=localhost,127.0.0.1,::1 +$ make student-test 'PYTEST_ARGS=-q -k closed_idle_connection' +.venv/bin/pytest -q -k closed_idle_connection +F [100%] +=================================== FAILURES =================================== +_________ test_closed_idle_connection_is_replaced_before_registration __________ + + async def test_closed_idle_connection_is_replaced_before_registration() -> None: + async with running_app(db_pool_max_size=1) as http: + database = await resolve(http, Database) + async with database.session() as session: + pid = await session.scalar(select(func.pg_backend_pid())) + async with sql_connection() as connection: + assert await connection.scalar(select(func.pg_terminate_backend(pid, 1000))) + body = new_user("reconnect") + response = await http.post("/api/v1/users", json=body) +> assert response.status_code == 201 +E assert 503 == 201 +E + where 503 = .status_code + +student-tests/api/test_pool.py:78: AssertionError +=========================== short test summary info ============================ +FAILED student-tests/api/test_pool.py::test_closed_idle_connection_is_replaced_before_registration +1 failed, 282 deselected in 1.84s +make: *** [student-test] Error 1 + +exit_code: 2 elapsed_seconds: 2.461 diff --git a/evidence/grade3-student-tests-interrupted.txt b/evidence/grade3-student-tests-interrupted.txt new file mode 100644 index 00000000..5ed40c9a --- /dev/null +++ b/evidence/grade3-student-tests-interrupted.txt @@ -0,0 +1,69 @@ +date: 2026-11-05T18:47:00.548192+00:00 commit: 7fc911c332aefe33285ea9e7811c37abd9bb20a6 +environment: COMPOSE_PROJECT_NAME=mtusi2026lab3-grade3-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 NO_PROXY=localhost,127.0.0.1,::1 +$ make student-test PYTEST_ARGS=-vv +.venv/bin/pytest -vv +============================= test session starts ============================== +platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /.venv/bin/python +cachedir: .pytest_cache +rootdir: +configfile: pyproject.toml +testpaths: student-tests +plugins: asyncio-1.4.0, anyio-4.15.1 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function +collecting ... collected 282 items / 12 deselected / 270 selected + +student-tests/api/test_access.py::test_outsider_gets_404_on_chat PASSED [ 0%] +student-tests/api/test_access.py::test_unknown_chat_is_indistinguishable_from_foreign PASSED [ 0%] +student-tests/api/test_access.py::test_outsider_cannot_write PASSED [ 1%] +student-tests/api/test_access.py::test_outsider_cannot_read_history PASSED [ 1%] +student-tests/api/test_access.py::test_access_checked_before_body PASSED [ 1%] +student-tests/api/test_access.py::test_chat_list_hides_foreign_chats PASSED [ 2%] +student-tests/api/test_access.py::test_403_is_never_used PASSED [ 2%] +student-tests/api/test_access.py::test_request_shape_is_checked_before_access PASSED [ 2%] +student-tests/api/test_access.py::test_outsider_does_not_wait_for_members_lock PASSED [ 3%] +student-tests/api/test_bounded_contexts.py::test_identity_does_not_know_about_messaging PASSED [ 3%] +student-tests/api/test_bounded_contexts.py::test_domains_do_not_import_infrastructure PASSED [ 4%] +student-tests/api/test_bounded_contexts.py::test_core_layers_do_not_import_pydantic PASSED [ 4%] +student-tests/api/test_bounded_contexts.py::test_shared_knows_nothing_about_zones PASSED [ 4%] +student-tests/api/test_bounded_contexts.py::test_shared_core_is_free_of_transport PASSED [ 5%] +student-tests/api/test_bounded_contexts.py::test_shared_core_holds_only_platform PASSED [ 5%] +student-tests/api/test_bounded_contexts.py::test_shared_entrypoints_hold_only_transport PASSED [ 5%] +student-tests/api/test_bounded_contexts.py::test_every_zone_owns_its_errors_events_and_commands PASSED [ 6%] +student-tests/api/test_bounded_contexts.py::test_every_zone_owns_its_http_layer PASSED [ 6%] +student-tests/api/test_bounded_contexts.py::test_only_main_knows_both_zones PASSED [ 7%] +student-tests/api/test_bounded_contexts.py::test_application_has_no_separate_entrypoints_root PASSED [ 7%] +student-tests/api/test_bounded_contexts.py::test_shared_source_mentions_no_domain_vocabulary PASSED [ 7%] +student-tests/api/test_bounded_contexts.py::test_shared_declares_no_concrete_commands_or_events PASSED [ 8%] +student-tests/api/test_bounded_contexts.py::test_shared_has_no_test_doubles PASSED [ 8%] +student-tests/api/test_bounded_contexts.py::test_zone_errors_do_not_leak_transport_codes PASSED [ 8%] +student-tests/api/test_bounded_contexts.py::test_sql_of_each_zone_reads_only_its_tables PASSED [ 9%] +student-tests/api/test_bounded_contexts.py::test_messaging_does_not_import_identity_at_all PASSED [ 9%] +student-tests/api/test_bounded_contexts.py::test_sqlalchemy_stays_in_infrastructure_and_asyncpg_is_only_a_driver PASSED [ 10%] +student-tests/api/test_cache.py::test_second_check_is_served_from_cache PASSED [ 10%] +student-tests/api/test_cache.py::test_logout_writes_tombstone_before_database PASSED [ 10%] +student-tests/api/test_cache.py::test_logout_rejected_by_database_changes_nothing PASSED [ 11%] +student-tests/api/test_cache.py::test_fill_racing_with_logout_does_not_revive_token PASSED [ 11%] +student-tests/api/test_cache.py::test_flushall_loses_nothing PASSED [ 11%] +student-tests/api/test_cache.py::test_unreachable_cache_is_bypassed_without_anonymous_access PASSED [ 12%] +student-tests/api/test_concurrency.py::test_hundred_messages_from_ten_clients PASSED [ 12%] +student-tests/api/test_concurrency.py::test_history_follows_the_clock_but_never_moves_back PASSED [ 12%] +student-tests/api/test_concurrency.py::test_parallel_registration_keeps_username_unique PASSED [ 13%] +student-tests/api/test_concurrency.py::test_parallel_chat_creation_is_atomic PASSED [ 13%] +student-tests/api/test_consistency.py::test_registration_is_visible_to_chats_immediately PASSED [ 14%] +student-tests/api/test_consistency.py::test_parallel_registrations_are_all_usable_as_members PASSED [ 14%] +student-tests/api/test_consistency.py::test_rejected_registration_leaves_no_trace PASSED [ 14%] +student-tests/api/test_consistency.py::test_failed_chat_creation_rolls_back_both_tables PASSED [ 15%] +student-tests/api/test_consistency.py::test_rejected_message_does_not_enter_history PASSED [ 15%] +student-tests/api/test_consistency.py::test_confirmed_write_is_visible_to_next_read PASSED [ 15%] +student-tests/api/test_consistency.py::test_parallel_writes_across_zones_keep_counts_consistent PASSED [ 16%] +student-tests/api/test_consistency.py::test_chat_membership_survives_parallel_reads_and_writes PASSED [ 16%] +student-tests/api/test_drain.py::test_draining_replica_is_not_ready_but_alive PASSED [ 17%] +student-tests/api/test_drain.py::test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! KeyboardInterrupt !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +/student-tests/api/test_drain.py:101: KeyboardInterrupt +(to show a full traceback on KeyboardInterrupt use --full-trace) +================ 46 passed, 12 deselected in 275.91s (0:04:35) ================= +make: *** [student-test] Error 2 + +exit_code: 2 elapsed_seconds: 276.378 diff --git a/evidence/grade3-student-tests-startup.txt.gz b/evidence/grade3-student-tests-startup.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..3855cc9bb510a49fdcd4c8b69837863673d47194 GIT binary patch literal 37486 zcmcFr^+Qxm)Rs_6N@*ztL1hI*q(e|dUdp5e76}ogyH=!>78F#<0`yfR7U^D>5NU~( zW(kpI=?%WQ7ks~e;QM~L^UT@joVhb|=G>WcF9^lZ(~ot|-#@VH;DNNYJAc;G%iGq@ z#p|q>ou`*4s3ASPyxkJ!!hq4KOXYV9@s1T@5Ja%m6fP>Hh)=&*p zX2@`0G5PnbgR*kg19ksO!c@PZU@+M>aIRI`N_A(VWw)}LyuRU+U#hOCCfHuPxDcdx zH7jU+GG=%oBiArHEf^H)@VB>zjma2b9XJCxYRx39HK=3GFZ;gfiN-$nxNnZCkN$G zJ1aPcrP0yB0HLJaAXfznX%Z2rL7GDUn{T;MWU7?LERJWZQqSlJc)sN4vt;`-OGq!L zsagZIgH%`y-Y{KS$B|A|?XL8-FZLxzRR_&x1%Lf|G-)X#)_|3@Lv0WjQ1Mm3ij?28 zX1m%p6Jd*~Tw2)NOz5t6+wpdlU`@2GEtk*Go5tl6lGd?FgXLUB7%gd6@9GBB6W(ZE`1u54K1vv`)W z;l1Q1b2W>sNJ6hK7`j@i&NFvc_h1}Q^9lB|7`t%WM8P#fZNjAW)*#92mDszMNzs=J zh6ddZbh&u4|GmynQtd{cCZ8j-ue>YZ!>k}(ot!?EZk-cY6O4FZr(l@iP~8!zoS=CI z?X{BH5tvXjFnbnN<(ps7n6+sbndP86x3mvOh#!p;7g}X&aZ5Se6gH0d(mdY4s?v ztDB?QVYX6nPHFS{hg#j3@pcas85trzYbHn?lOmGmPhhmyMHXGu<1@}1j7IgUWkj?& zeBI{K9}pgqB@-PUeWv|o}FK*AdKKtXpbXW}8i>55H&J;2y;n)16GTIvii+2}K zY~a@mH^p-C8qH^7c?oN!+uv_Qu1sOJao8IZ_UUUF|rOSXE|y> z$y8Y&!roN2M_ml><)b`~?xj!bs+ZMIZqbm*+Gs+T@o|r5IrCBO$?*MXxF@fq+wL@w zcbRqRWoCw6BwqS;W5i@@S>El(mGh>3eL6f2m;=XBk_>W_JIBWj-CvU3gGN`}x0e3# zRjJO45NMkb-xe!DE`K(i5G)*cwb9V$U=Dmx2N<{MkeKJ=dz`nbRNq6Vv8>PA#sRQW;Y%Qsq#va>E`aguO1p ze6%s>N#T2`jrB5y4kGsU_8&1O#e7EY3wIUYO2z1$8YFk!!%T%gpH0wIRaMh%h$SHm zK1%k+Iwy2Yr-(pNeC$UjjYU11&BBO@2-0pnoeD;in z@0sZbsgL5tq6+`9MStx)n2fKM#xMq>Uz|9+^Tv1wt*>d=V})*UE%Nmdlr%_cjGFb& ztDv_!M^(w8#%~du1MSvr_By|#Wu)CMKR+}X6C{zy^e_&o=O=xcrrR> zN+9Bad_4EIB)h{0cioq{$Y>=#uE=-OZwUT=FP7r26kh#-`99yoU{%CzbUdq0&~PNm zZgJh%J~S@%XzL|~W=3+lV5F7-;m?YszM;MQBNc(ogr3eX80n5`ldD)nLBMfbtXk~w z_6^OMH4LRdM!r&P6=7=%dhF0T&{-^FMa-#n)!apxaL zk%&n=8%N1Vk7)@u;%(uJm!m;CGnF1ltma3*tkjz2H1gW?IO`7PI=?g1urza;Ne%DR z&8NDzZE|fYjH+2;h9_*Q=-C_W1Gm}GE7V0dg?jL1x=kAEH)<0u@wPCIwsE_SjK*kG z)~aWO(+oPtrz{*Be~{}g=1p0Q+5TyOO)m&~VB>(0OMf@6Bcq(rc-_0Cq8EEcFy>CC zVV)gx4wA3L<1iO0mS$VFP3#E{xjgZD7jf+7&bxx2Y2lg7td~j!rAoVJZ8Kesk>I-O@3-_=2`zV!A!?aYxsI)Eh;|{P_7T}w?#5rZ*%4=y z(qRGiqCrEnL9?a*)4SN>d5<4u`Pf5NPXcO0=IuoM2XKsGT47^v77Q?jI#-Fp&!Rd- zl9LJe*LE39)v^+|D2G-I8{Yf+pK{M!7iNx9f8*E??p>vX{P@E9PQ0>O+rQ*-q#VoP z)kB$?cV4}{MSFOOA-PRLq+KV|^aW=;!Avp6+o{lQNX3n7s^9eUS$tLH1iQ8KkVz$1 zt19d62b*alKD7iM!^A1VCujGiZnSea(?V^LuWWp}$jp*a35z5rv%fs8Jt_$T|&U+zoA$Di$-Q{8Osq0yvLER+}*Tt1xt)3O7p3l$t z13Y4#o0wuM{YhG1aJNI}-NXzu2ovt^_bFHW9cV85SaY+^6)~x9kKYYxuR!tq^Gk~q z?SI>mv{srt?$y<|J@wAnxyHYa;OO-uSWTc&r0L!4aj69F zgfn8Q8276(9Qf9QZi>a??tNJa?_y34OV4AUI`ZRPB>v1Q@DH*~T$WRQcZ0LkhU;d^ z1GNig7CEo%wAqtaRGj$=E%faPW$(@E6QZJG-i7zFZ@aBGBV`2MyQe)dN8TfsX?|0*{217cb>BQw4~{p#;w;gM6S^Gi{|JED~) z>FMFm@C+$9kGZs*-BqM4Bt9F|-i~o357Ci{#~1 z;tXmbBa6#K@CPehEt!Qkh&Zu8kgqNsrHo?<1daexv(f5+ZTz78C}nwdLCag~I|7%8J)N;Vm4**+M*e0vs_R>=F|MO(OTnwd;@sP+UtZ{sn1#g1D9@3%NDr2B zQ>84)vxXhDwV(JI+n%M)`E8sYowlqA7!2I*iQ%rjS2yT#Sj)knYgyM>@mjz^zfwa4QK+#Q!1P(g%09Vbmk zAGkb^D^L1K?eI)~>Rm6dNqT*o(aro9zOI&HT!X)-)?910vNfNGPdmiid>g~z!tW`k z!jk-kq7disqr-~r&zxv`de8ismI8G?iU?T|ts#Lk*6WeC=pLM%6=VADG`!-t6(V&f zJ*c()HsTAHEjQSv$Xi}iVd2SjRBYs1XIJ>aNR>J|`G8WPz`O~MpPO4@ta&InX1y*vNFG@l>yKi{v9Qnec z+%|F6*k6h{YOyB^RX*j1YNnu%T=@3sg=}wZenZ=g;K2~V*Y(`s3?d1}7B8zsrrtj##4HLTKu zv(vF@S_W4z&CU}ilXRnAAGgGc=Ez$53_HrI2)1{4*Dbc=V@R(W2h|x>OR6$C&19Ga zLNX1-xHTli*t0b}!;&3*l@@DRo3?mQnPEpnkJTQ=~Y>3mClUEHsu z{nN2oB026mNPp^?6ZY8Hw;LN=k>k!wzZ3o(RC{8tc#LA)o^v^F^^JN!>HEu3*ufN)6AF;x%;#zTB{tcW}~Ux(=kVUTjYvYYGE<(70pE5i2c`BxYt z_-%x#63IvrcrC{qyu#ZVQ~Tx1PERy%V~eef*##wSva}s*m<1`A=efPm#MG3@!Apv! zAuM)k7VLuSVGg&YLJpr{Dmd^6?Ufn)ub+%yE^#~J1NzUIvz7IaCTPVC{@Bmto!8fPR5vQvdeg^w z>bAPQTt-%kEXgC&xKEZd`K8TT&x-UQPE9XoDJj`c>tvN5j#<&c^4{S5C2dowCS`<0 z4ef?9qR{X3ykoCkb{R5mW_DaQll+AH`9#x4BS|qVc1no4#V3Vylttd;>Y)r{HAXgN z$CNnRR^FMaU6=Y>C`$u1*CM$%uHqN8`H0;60XI$BWkT%c>;g!(nO|*Eu2%gqcS5#O zB!2`);R$g8ifsJ0$v?|(+D9TX1*?#x20HzA)@IK%0jY?{g%z8%5VLa`6?_5P8_tN> zYv)4Dr1%E}3>p(6<(Vs-LlI*(4$bXX#lw&<%&qtX)$(4fSum$r8LM6STRBfV0?%Dq zCnjGf(ce}Hw>Vy7D3N1*yF629+e$Iok?Wj!k3#fPM*C{_i+DQR44R;vG}+HFvF0HA zPm_erm?M_<&mSAfJMs!y>X$QAn2CNQKjNRxu2X)yvLW9bYDnqLI{X_G1y9$fvdcB! z)y-r%1q=DiW7TRVJ6kK)5F)7S=aSFxs>V;yJ)a?TZ7UQNSj)$jhL3W4E8TjO+%t>h zb44YhxXupxx$vl`QS5VR3CZu=+f!ZnxTz#^^kNjdw5nQ@-}mK5LcN6p6RzV&;3U8FO7bw_aZKkVG3yDPJdHR_Vw%Y}9x_|2HC={HOGO z%Zm1=!9PWgEwXF~sfUnmdnk?WZezM^pM|8LOem>l4DuFrE7BojGBLG}#%s!y@}DG6 zcjz|0ZN<;e+sgj1Z*Qr0IO#=t)g zE1i?GE?eVx$pu%`_QIrVLWtR&=3ZP!4iiq9;z23ZwrblDTxchvb`sFLOQTt;#4_EX zkQH)mR>=DB;>IXtVbo#NaF>@+X@5RAX_o zw5Fxa4YiRIMQ1N4uaKUmTpKmHBkJZI-KP+$b0ul_=^e4SKljmNGf|h{Y-GO2*JBw9 zW0ig~Wm-v^Dl#20+>QJ=(DHqzaYEdbyK92>PwS`iHR~&ql1+LBJ1ZlnXF}UK8}%nR zzvS9wC7tDuw){-&Eg#q!K;j>xXFnMyy71)xiT?ROTcvi)m%Z$M<85^}pYo**jM}fA ztD(Ln4Mtr;S&gkF=5%!CqmlW&*|+q6ze3nK%n9udoZ4FT)JM9i9@F0myKv0nb~A~~ z6PwJB6Q@n3Y1a%U4@n*FyM6QP&hdO=#rnw*Wxhpjx#fRx*U9UTudBy!>$@s0qtu5- zf~zvfm8oQ+9m&^Me3zUj<#NHg_wt7j42J}1;ahARd9+FB^kOkj`F*pma;qLbIu7;|!vYww0npav$$`hrXEx9safyJ}L-e5KrFq5ZYn z{rczC)yY=9zPFR}e^CXVv?{Ghyl~cIuyASG$vf^}`|z**r<_-_cL)76W_*KUHdC4x z-w_EFoF89xU3il9{0GhM&gO1AN_|!Bx&94b!pHls$Z7f}zMHk}6m&9qX2oS=>D_Z< z)fP(CYTB?=(Nb;I)*POkkVG_GS!ATEtPHGO^8GZ|Jz3`5i~6Q}N4)2QeS5-cfIj}L z6mEJ5Pa66s@`7Yg%cIaC#8ktH)lcOyX``0koIGxo1~?YI*Cn*RUpXgL%8oK++Pp$u zseJxWpQU(}seSwTG;Z4=Ba-_>(7OWv5N@H3YV7N#uG8%aJNW07GFB`jldqi|%T~>s z&c5kdZBL4kz{@VOh}uSr*F$VpBT^L!gB#Fc(DDspo5`KvkNkS9o;f+5z;#Q z!&N_uNbnQd_FE!{_}uLZIL^)e$@m&(Rla5ZR4n~*($*zZMHyyc{*8>XHhb8bu+?-J&sfih!|0l&LE~)yW=a%q zPmg=m>EYEvU5tiFbG>-WZnV^>nSV+5uIY&0ITS{GhANWoC!QQ>`zjv<_u{15nW%NyD+N;${)Kg=W0YqQfXHtDkWbgr2uPdXKG?iPLZ^XXr0 zF*c1#&)>W+{`nh@@z|v!C_%xjSbJ8rvn2sV7C$_epGa9v@)}La245>FM(2sDha0L@ zZ?<_1wk#XE`UE5y2nAIpkcD#oMxTQ>7icOt{%pG&+99C*BV5eKm z-z!XDM-oQu3VqR?Gpu(XTa=*`XI4I@=`Yj7z201NSRu* zWA)Z@6xq*BeKQfet%7r#{u`#~4#QC?qeTLlRGT%$+D@D=t5ts)+?TNVGq8e)Q#c%$ zYHm~Bz3rELJ4JiYKRL=O?~5%ZTIK7t0Y)dHx}@uPtyPI~Ld~hx4V&e7-l&%|K?>17 zc5R=v_s|h2^LM5gp9KZadw(xkj7bYZ6%u+qswM4eD_+s%y2lJLhLLCQQC{DFent+f z@H+hDS^2Uc-PkO#*|UoWUOjkpo64u?K#XyI6{=jdLwqH((h;?6y^{VW59=hfA?ad} zlCX^{vf2Lb+PdB)Pg2Pb!l07!8vTntF=ym=Y;VZM{<<)%&TKcXk*=>YkEN5c$L7xZ ztQVH%#}I{3nXYHiSf!@hl-1en3~?9jH%f~gsMd`&XZebv(r0=78Si(o_@dxlYtoq2 zP7el$8L1_;RC&h5N6c>X(~L8H@cmoUY}&s9sD!b=X$}F;d zG%b!6=khi19Bnv{>6Y}0zymw(K*GJ17EQ+y{jl*hFGhMZHem7>^K_$t+sVNu30&q(vTvhKMb=lN zQx@*1J0$<7bGh!Td`l?qk<8$dRm%*$RL4nX@_Op_z-GRGv+I{`K%p&irX@Y~{0YA3!4N|G}pU8n8hn1VJ;$r3x=}5hjSk$qa zJRgqu^xUz1Tvos7jYKj|XBYGKo_+8&=UKD6tN$FqT`b+ouE-AS7MLgDsXQ`-a;1J>2p}=t-&vnm^j+DT*;$7s_rg6ghr_GE($F8Qqn8Npo ztZxMP zlu|ce)%bDRYw@2~uHEDouLAE(`@=Uq?(IHZCE19NqM1w$ddP`)`!ZHWRHGaVmF2Ah zRCH4wm=12=x?lJ#>COe>>#kn{WA9{J@KUuFYeQ2J`MAzi!Ozvf50VCZr21Hnw_r0% z>#1uC{5M!TuAt?1_=}7iZSxL(apGhy`%SO#qy<^{=-cxg#6YRH)Hv@ zKl$c^z1%eZHOCzWJ9;v_MP{sd?EQ!Y0u{+>BekiRh+#9>u&r zLz*REdlJ!vI>RsR)QhX2&YasZ9dE~Lo4(@yn77e2 zf%tCpJ8~<_bU=4ALC`ioF3Yqg$4YQxhA``ZbfQ0o9ZOyAUYT4XPX~Ji&h7N}c%L70 zW@8KtQIqeO!8pzN<{f3-PFdc@mL)MqR9Xd>yP%r6tG6y4>9_GwYYTP@(~{J*4Xz?% z-|~HtlsguR4*X>&q`DNb%Y<r@5K_ZTRv1#w$dwW&s5`Kh@~M=O zKk00)W~i<;XqkpoSH0|=*QlzY%u_rZDup_DrfQ~eWIqE%l8wLVBtVYpAy*vvLX^|u zGRaO`4s%UHRzaykqibS;*Ux^FXdSFwa0U_4Hc)zCR$MY4yy~9-{$6V`{n4h@haIbgnyBL$<0EL| zcr9|Ma(A|RE3sT5J2vVgYaqE^*z#wP8)~MD+k9{wrF=YJJE62xC^$f>5<&dDncrfF zA9q#Hai@Kym(_Iskko|LgwE6Ptz}amq4$;{Q6J~WO69L^8gSR{PFsfjs_N_>J^O{m zA!@hv=3=Xu$z*Vs>^8m{pZh&?!P!-PUSJc|ijp@eKl$^XUt;5d4nc0qq~V3JiQE2( z;t5$UsDOO8Tig}$UmPiI`e;f@>2u9}=_$)SB@>rh9Zt#uFN?i(WCAtqx|4;B7-cpLP_H3xv<;R@h5X=7H^4jCNMTz?*;1>E z;(p$bf1Xk%bE)>H>~7KT@B$~x&mrUbz08~Izrs_}tej(}8L++nDGLhKS`QU0K2L5f z|MPmQVoP|ko;Z#OH}iiGv}(JALk~Pdv5ZD`NvU4VH5RcATtQduaL7MirO*0O{2g&; zl*4H!$GJ`Ft55TL9v{J9)3yO>7nH5OYl-2~vp2`KJr-A&NoL=DaUvbsvt$10V;0~| zg<$qfU9w0mWt&0kt}*wkWI7H9bKJ0aHk_KwYHA#u!Cp+LM0T{>zmaT|Yiob2P4Wx4 zp?f(m>*&%qv~)I8t+5?zs&BUWh`zsqVE6+uTQ8+_0lV{Q^Oq*vnHZxkSKp5uK=7nB zixoZDEgih9lqh!ZiP&qJ*^5dARffMWDkZYq>ZrWcFIg zST+Fqp5O{JsxQYt97qfLUDE`BDjI=-dASIpY5F8a7C~A)%Q!dm-ff-CCQM6DrTLIe zShkZ<_5g&e{j)BDI_-x{-;bG>9_mLUUYqKsB>LYUAm#_ zK$XT}3Ns|SZ6GN`c1oCNOM3=@eA{zaW8)EG-wIqXApvpOz?Hs?g=S-O^U`daA#vbTJXs8Q?A^LtTU!oQ_W+DMW_5_zwVS z)6iAaI#7{o-wIqr6aeBtVTVng$RUM4RwGE1i5h^6F?{l(NYH)gS)xPyM5Kc;J0A%rJS%1xo?NB#^^(fJPfCs@+2*Mt85CDBoa0MFGj7P-Z|r%VsE}{3q+Z zJ8RXN^(e``wt9}ry87ZRVa}@uZ-m|w*6L)`x(FdK#@9fd6CUH6p;;FfDmHoC{(BJB}n#qA_{b#Vj)JsiL#=}! zo%7oo_N~BFHZGuU-bfwwRQB+U1NWJ`UDpoeNR^sZpYXhX)OXqut#%6Bm?CtG7`b(XRk=xaJXW?UOaUC_e%?Vte?Yb{(N(Caz zJ-Fi6LwBC$Xzav#YrCn7C9CAuAK4dJ*`sw)eXZXc5?n`t{lQO{TGTf~nhoeVw5AZs z47_rA(%qTiEESz<yyDu)N}VBJo9HM<)1$#R)p!0Fhj1Z1>->mKW4A;OUdK<~E#;)` zy?WtB+*Gh8xZZjfFp?c$a6#^L(Jc`mH+ruHNg>!Lh)t6}4S>rl30Pwx+YbI=|97qi z0Pq4I8^3;bmwsYgYrhg}9R3i4Pj2xN>_{@iI0pn&WDOscntgQtGg?gVE(I$q6Q?KA z(|pnE*F|XvnT&`-b=iMs?)*EWcQ^S&E5x{~Wd=(jxNyD7KN$dgPC2YQU}@60Zw00v zM*#Ja=Z+RJe3%beV(%!XLEk){(qyJH?-ZIFU%i{Ak6)M2l%2M^EYfvV_DCG<<=lg` zFaq5N^~-zRmj&I28KC=+6e90+9{|vOSYyG?Zr=)Aa5VrTKI2K_*Z1ctC(btu{!CXb z+CDW|>}Nwz*K}wz(zu(Zh4-GTKR<1yBGS=(el?bKDVIzGqp9EqP*;IlZ$A%OhZ#oi zv5*uZ@3js9XdTvA(6ZaN0vB8jfOywQ^{!7CrhrK>TqN4ogU!%7)?wvPlmH>tsXxr& zN3GwL%%={MyxUO*L^>F?62n2=|H^um?s~d|^9@^ni1GPq3@nA9YP7uT4FHB@jbV+! znD6^mU@9Vfir2rfk`Vj#BEHL3_Jv)OK}*XA^#l=fZq>Uj_s3$lx!WI~pNLfup>#S@ z9zy7@f7TeNx5J?i9}7Ys%;0n!holf04t>!8q)kItg}Z_Jy!%$*BBB5g{j}3p_`dU0 zxCZ#h@W#)eW2vW)6il+jEzHFBe`f4`fx5ouwKv!@jM!i|1k*hcyD9@a|iIi--b19B9zKmReZ%Zkr^^yWY|5IL`3%PEQ>7Ck^0RvLbC)&v(bJ zUCLdjff4X*P?z2FY{<7@2Jmc13X%7G3jp9-u*SkkyL~Hg!PNkWc!B$k>zU^QuXRQr)s-9kF$8H#>u=ls>)qV|aBU5b`WQmm**%T+ZmGqpqrffWtaW#APp3Rbi( z!~iQAmO>C-(ExxI4eJhA8eG}80#lD8fci*;tpy#=`J2x-%{+c)oy6;WXm+n<82smV z=Pa(pmM5!ECFAn(UZ0i$+Xn-Uq?iY`$8c9lo<{sGjLHhsK4;#%?1e+1RaBe$pSiGIXm*rw4K41L=f8 zzDRN5^#7;#`H9QBScKfc^K&P7w1-@Hksevz)*x#c^wml}aaaoZ3EXi2K*wQ?lGNnCR$xD(haAUOl((O^5f#4JJo5wX z*DZ1R>J`V`J1W8Fz*mns$N&zXmrR`lHZ3New+k9cOithU3a@--5Bn_}RtF*A1qGn) z1baat@B%J~0lWZ~LJ;->ECBkR;0iRVOAhZ_fvK7%K;4smwf~^=kJ`E`1?*0vc?+hY-J;eBYl?j$Y5RUi& zfQS!k41Q|ew*pfU;Rlb^H;x{YGaAZucYnq{Bjr#ihV{hvNJnL6E;{^!FnGnq6i2l8 z8O4OiykNn|R`lCXl#N$}CTw$W(W^a57!x_w<#>t;LLeGG1obUA8ipl+Xb3Yn9YY~0 zM24dwBLLtq1#1MZ^X^-Li--b1^waM6_$A<$H_sMk?UXW}YqQ=oU7R&3Y<5y1YjVx# z(NHCKf;ayCWMy{Bzr1#D815q5A6WYwIYC|bme`sfk#X}Y@^d6zmgw%>1fxmPY|C!) zu0!IaYuPmv@=`gUh>H>i8@e;m?=E&u!bK6h1>wxb`z??CQy24KHY*UtV2+a0TU#9$ zmiZboG{@D`S=Jw3-eabnJT)^f+>~}Hn>_m)!O&B&*Wf+OO5g|@TW4yl{{w&gEu1x`zt|HAbhaR~{i>5w#RN{=j@ ztIsvx_VJmz+&^`9HSX$2@`osw-2)zXPx8oZ$ULnKcxHKsi@-%iP4$RxW}$+a{^3Po zHhOSE09V?*6M`@}AsnO*y(kP$2(T1_@Cg9`a6$lAHhSm$@6-?n(t>_x;{srbp7u-6 z!*?yn4{a?`jwU}5FAd{2?FpTibZW*$XB@4VsGSb|cdj`1$Vi$RK7jx?g%2&z#)O{< zxG8)F0dD&DD6{9LfD7Cd));g`!zY{lqYfe*oM`Hlk)i#s>l}BSNwA znw{@n*r{@vh;#5gKJ+&9o-p6zBgK!7yoL~XUP1Lk@FKluoqQq&VpwaL!BPmqixhyd zoN`!qz%nah-wI4UjsWT-2bnFzX;?Ld#Cj+pS5LGUm?45jt&@~%M9+k8I?7~K={8m0 z=NmZ1#ZOPy5b7q(Pk*$S`e+@5z#Tw!I`|IUz#T9_3~&dq6oT*_008a))*Y~Xb8z1Z zOg)YO>LXqF@ch8f%k{s#e0~bWB`LI31flDm=dUXi$R0R&mFmVr4&j4WnTs`bdMv0-+0-wIqXApvoD(S@sw<*;BzJYA18 z?n$qj5o$s2~r9o2R1&&BHz| z(fyx$?==s&pm|th<6p#IzlYqP2?^AfSuT#5G4PZm7OaTTC3oFhMW%$SV3vgrEsAoC zO4CN-JUmqVv)YsV1mF2G(>rs-{+E_wFnhxQC$PXGgws^;FSyfkg86C39+R~pU}A=>mhATruZbs03Ny2gO~kMbHvf21{^YO zoQ*TJQ9At>N|``{p7}CO*F{-2=F1Fb8VsEf0`~ybY2bU11^1u@F+d=Ir4WSg0RV6h zu{hvvQ&n99Zl)CV_3$31_R3RS3%uH03P3B(6^s0>*JC%Q57m5DL3Hq)#=R1s!v zW-V@DU4{_wNT^N&d!!2R$X18}JQ9{d5cWs_fJef*1D4OQ`&MA;aRg8wiJ-F(r(*J5 zX9~0urE}LjDmVXOM>9i74Sbh9t|E|e$M;j6SmD4$xkP4L8r;Q{*PJO2Aq1t~KwWMx z^+p4H0A_&H8zhCud#N`7KFPVfme)2)UE2@5qJEp9lv3?a~UsJ<5hf}pNL4A6C03cYf$Wg7@%QT3PHGG06@dA?ttaXgZoxs>Tv{6A7Nm$5T^#; z|96Jqj-m*vYFtILwz98IuKd(0l7HeLGa>5WGfWe4Ec`z&x&FxL8a3=sAn?O32nT*< z5cpw_0)hXpW9|ihzy*OH))<^a!+yH&xQKAzpLNO=R%2gFnPR`HA;fs~iipIBCtg0t ziBry6g*88Gt*pI;|0cTFMD86Fse@BmV1EeZ#`gAyP$m$1*nmvn-{;C+CJ=hAfc+t? zQ7!l10|}yPngDfYUZx+6zt$9fc&q7Qx+m0lz^m=P@U}pHpFvSe=bFreeTTyj+Ju@2 zb3Hz2Lz`z!3nQ>YhU%g44w(z=kYNVcA;VHY+1nul0Cvc*MoGUta^DJ6n9#=n^;%^n z3p#2%h26~c+4-7+xdlUUwCQR>r#gD_&a2Guwka=_ENq@VS4nm!rkatYo zd?(6;loa7)&Ny@g-5cHs zgv84n@pqkJ?&61$GVQ)8;V079v@x5r>m6KQ{vWfWC|9nscAECcD5RQOdTe({RkhWC zmq!F!rA&*M`L;e~)hIJUr+t=EnLY4oJt=UL>$kYkFKO-VyOf^I@!V^J=rh5)GRG=L zTMv(-iR?_ao*9-Tg7gnn#)OJkKPf;qvf#|J|qBdJ%h#>YQIirQN2n$kvdG0SW zr4l-!oa=O7EKyHBS>}8Zw2*Dn?;7k$E4$rnqDxi3c)G1 zwk_6Yr<4~@AJ=rB?FcX(-db5m6RHUe^!~g(Gs24F4LebuZs%|s^G*IRmi@b>+iSNi z>EKi=?KBpQ7#sJEBWnxSc|;q;k{g11?6s4pc9!{_jjk09N7`BmLU+**!1&{!H+d*0aH} z>gpBop-$KRsyzNBZj2JOi26;5@@C~9-d&CsK0sAZ#l}WUOQmrU6t_Bh3DFB|v{gB! zaNQz}i=LkP(L-Tj`n_lY0EiagN>5#4d^Z&O;qUMCOn~T1PbI-IwuX3_v51|^51f@{ zxMZvpTgvi7a&+yFX5GKS3c@UpX(k>j2&;86sy&1d2o7VQ&a#)igI*5743NEpq!4*8 zH~;{G1FR9K&$Vv_EYpF~9$)?~TLYS$Uy6EBjMTz+XFak+xP)EFm-Vn4?gG3_C07+^{ z3X%5`i2#5^BCN4+&~)DlTyQl2;+TM0oi~TutISE?l~%dOwt^0quA5<3g;It-*-_;) zJc4`%_9U9TF&Y?w0f#*Y9&i>g;4lL`8~?i1-hcxx7;splq~G>Ge*^+B;J~Ucma5;h z?#Db?Db2QjP5gMTqs*VIh=7cRc&4fT%1oNRXqx}FGPHRYY2i%|XdB)q!EJMZw&86L zX#4N(X|HX-1#QC`CC2s|@V08t7LYv1%3nS1pRNvbYu!E*y251_V0?J;~0Dz`pjgnM5_I)c* z0rJ&A-FNHf*com=mBZP-K`(e~3WN-LtO);jZIhj}#$FxZ5kB;o%H|=Duv{mj+(QU~ z6>bgchu{@{7h2&kgVPZSNg?vy3I_nJa9AVoEZ4pjxQHkKM8DL-Zh2Jm2@`BwVk)U@ zqaA~_Yky?wqIv#v2<1*jaHts6EkcvuPF19X$hoh<*?)37zkg&5l# zap)!m?nMb$!vkJr2;UIAtuuwb)`0FE?6Cjd=H3n)x=oOWhc&8&|LY$_)iePesS;r- zi>H#rS|aLaS6Wnf^}0=uP@g1FDrFfyY7h2Tj!^e8U54J7!*9}Q^RCgt2uu2*KA70(;qepneGMv?J7MhygkcOCbpFWwQYQpQykZ z)uJ)`R$!{82~c+)R;4z)LPnF`kIlaojGDcHLaCFUt+5huPon0Q*iFRQ{&wRkd$A@v z{B;31&%ynG&vUn+tPcDo0Z2LTe~_@B)d{^d2C_QPcnw&-()w@Yz-J1_q3;q###O05 zd0Fp(zu_-Dgv+B8eY0;7{gD!wI$CRV@CySRHQ>lVlV?o>#|SXga3p|-`VbguD8d{9 zL;W{Q>Do`6&SOJ;4=dRG*63 zw*pf&O@Mk8#(_ScCbVa*-+4W0piqAOVLxp>mHS10Ho9+AE1^cqR4@W3XHa*)2%ntk zKzx81;N%QT0c9^f0080xtTE_RpAxdgxwr^q@I7^9!1TrGT>68~~s>SYz;0(SNx%AOJ5CK&nld zI&^M{`GsS(t$Vt%Zo?IU&3=kBb=f;<{l^DOOR1^3`^32hq%ZtE6Lq5}f_96zz?tYb z_>!`V-u7lXbSBbMWXw5{wr!hp>I!AWYpHD_H|gnQL(a(K2@d&VgaGA;|M3yE^VMpI@8X{?}?v)ct@BGqHTg!F#Q5}pPwGbGva+o6r z^EG}CDy8UBcIa9NyMC0_2dfjkpZZ(ojHN=h7F}C}P+N0@StBZjH7LqX8)2y?n27n5 z+G14`OjwOdMeVF1brh9G10$0~h0+37oSlkxD(Q6Zq;!<2-FZ-_aM%`Wv$A71gOJas zOEsJBeicQJ%~qeYvm0+Jbp`epdf!Zf4<$idj?hQi~-r*iyf|2#URPQ6d94 z7r}LKE;`ywD-)_Gd~`4F0080+xHi+4aND1Qe)#)4eJddPZvMtI^7ygkP;0j!1ijXh z@yUDYA!p61bq>nAj%S+SWiL~IFs0gpXmrgfq zpDF_Rr>jh`6oT-(H2?->jbV*nb18`s2h!Rr$~O|rzvt36>540Ue|7SB_jSF6?}PnG z<=vYYj{<%KU2A#SNMXzz44-c@071P69Ay#x;Db4cV*Hc~mO>CdhyYOP_yyMZYK*Ag zGxJ}OSLZR8I0XKwK@>h3dvZv0{uo-pFcYgQ--EVQ+FmBKOI6PmTI4A#^Q+Kx)O$*n z_|DrX!*D8?8VHutq2mBc$A!~HFy;39gOC(r_rf0x0r23J1gx>Zskd+DZ(-5DGW5>8 z!82h@(T^oFY{tTQ=@-@qhF~;-U~F$RGr?$%LliKYuoQyuXaWF66V~`8@nYZ1-=bU@ z>)HRh9Qub034xXa;tfBo6Tc;MzcFjlXj!$iN?EZ=&gueL4gQFy` z04I*AcdcFfa?m?Xil!oL@wM{rcCDTGJMW_)_@&AaTv;k^Lf@ResRVgKh*B9{3`-#h zhxG^m^gY29XjHEkCw&*w*+fuJWx%Nn|KHskBEb7)Y3=jyQwWp z@YV{VSZbNUQV7D2xQW26k(_dH6|CvEycDr#ChBiNo*55xUv+!_!>(3;qL$Y-`W+G# zGG~+DQgmGPDr$Hs69fz05Kh@;Hxfyo91)=p^A)VP?3x%AhS!BSA%+?)&%FhW9 zv;-%VfER)}xzIPbcLTtA0j7Wx3M7TtdnXhCzzGG`2+ZTyGqd+QplxQonmy}!H&vqC z*^G^{Ix$$<5ttRU*|LOO|I%_`fM(;Ns&&UfO8lKMItb=h8Gg3t&&C>YQ@2S)yp}u1R zelwcEO_D#uAPD9ZT?r%XB z5BH5Q@6E2w2=9`duOf^<|AvGmD&I9w3Kg(2fqzmJ5Tm_*P4yOQucu?}-PDci5R_Ot z3a*#de?lLJ-pU7Wd>~5xiL4v7=&y$akb{HN<`IHV z*NngXYwKIky4^eLK9IB2B>%q0%dK`Z6%|OC_LtHtaoARTOnmca$=2cV;wa;2<%U776^T@^f;#;EJw;wDDp2TPb1j)Q~pgRZtD-UkT zjk`ua2}6=r|G`SYP%AZ3!wAd(R42hR@B_>M7Gi)IfTa+GXW*Rr4=@AZ%1Us`|Jtx` z0fgtK# z$8pc*blrJ@%5U)yv;eOcJe&Dt;MFVQ305ym0k0PzDa77ey#RpK3u^=pAK5dr_dB5Z zHrcvlB&RVaV4iBA_@fE)6ZdtFl)NRIY&yJAGQ_weUv!hw^}mM9-hNu%*tP$yzlGn@ zK!h*+*+1|m@bUjeCD@zZ18>rR-lzg^f~63Iy$Jx|O|ZtVPcQe){4FXLFI6baediy} zyI6tX({Ac=_u|i(TNJ2VY=__-|138zegSQ`pPpWzpD=~f5&R|q!@aBq;DW3Mtnoj! zUHK!F-}j#xhOti)DKQEqvJ_ckXrUyP_q(!FN<|1E>x{h=;*B;-T2!*7?8{J6$y!+| zTPk~YvgLc8JM{erKK)|OInV2y`#ksF=bn4dJ@?_QxRPmxc^zQQ0e1Nr&)xbwqYlJ$ zzaLq!yeX1e<=s`~`BAD>BYaz6lokJF`{7S^`At_}){VN(o40kpHatArf5`8$^=QEN zJ-42}>Mx%S%&}2l^WW{SnPelYA;A$#{@jdwSNV##>GqA)NBs|? z-JBxyFmbVT;3bp(wZim22&E3ESBJun(@CKoLS{!7inq?6zJCp5MIkwVVzm%ug``jj z)fSzn|GRy}!b|W~ksEocRI(@h{rz7%H$4eG<0SdYOUTpWFRqwY^5sVmL?=S;>h}w_ zP;MO9bJv8w&Qv0>d=rbW&0CSh4@I>sO?`j0d5fnxG@X?100fKBpTp=fB_C^JQuQA zwt}FfYEbtWUffLT4DGY^x&HEQvdK)%V$)n%Y+j0guh?wp?>qhvsWK07i2Cju0DNx} zHeHPP4bw%0`1f8gCdI(a=^_v?T|^o~lSh}Vpr2J0V6`KmvFyS(C!5z;9z*WU;?MpO zZN&=)nNkaBBsw95h9>4IEa}NCh=M=_Ow?1DXG%CB5P=X7h(J<+Fc$AKYxTsl~h_JcK4`wx8!whICKZF zrGVWSb4vj`WrPs0rGOq4Aj~ZVAYe-YX-s*^E?Gf8TsGKqnF%<1xN_!W^@VCOh2?+$ z2u9meY0L`8+Tkxe!bsl_ff4ED-Vw(8ERZ)D^@H|}=%G*)X%j?|&|^XriTS9sP0T0~ zJUT>?PzAJDMh5?>3b-yV3s&9334YmX#CZZ%ji@EqKk2Zez5+37RGBD7Er8|OU<{uRe>3|aV&dvSSA4zW`3`Z#_XD@C!MQ;~{M zut`A%9*;IDR$`kJ2(jwS3QUTDkvrc91Z3_b4R2w=B`fGBI17NDJp|`1cumoeehsbt zA&1l;W%ne`xdc}{@Ax}%v3v2C_U}lc)4@WW7zK+Cj9)~)Xr>H&5r!zc{0vDk5)Cfj zfq(!a(kLN?FIhoQ-D99`pSftJRp%kkqo@BqNnV>h|Hpe;{O}J{7N9)oM?CSU@V9e8 zNEvIs+lw$On!!p>)Bj_VIV%Dg%!-i4r|_)hd%>VGLD;=$O!+I1c8EP){Ey5i{Oz=Y zM!1KQrgBCR?y;HPVcUdJ(AhB_hdMhame__NptB<>Mxr?<5YX9?u7iAK%90fXCC7n! z(LJ2^QDN(C#FiI0MJhL$tcKamtdASi#gBV+pa6iBRR0){nHjF4Kk(q9QZ}V zUb2ts_>je3@|c$Mj9?U09>%ky%JYHBtHcmcc}R+psPce-%0s#inOzl2RuGgd2uX1J3V@;%47_y_UPeW8qM5O8&MVRXTx|>2pNOmCUF)77%AQEoVmoaj0Mv zAV(D}f>kgUZifm+QjBD-h^Qh^!6nu_rHkyGQgR5Anz^LJ9zKrN}=Mz@s702q^%|4wmWB01F-sX?!-wM*eom0m}w~ zpJl{tX1pIQ8vXXdlRg~{ha)U6ot0rI zZQ0v0R|bHEl>yS2GL=P`C2tIQ_*8EAKvml3~ zhV9m32w2G>DMm6UTR^~M3u%*R&>P0yW1(5sCQ`4t zZ#jSDgWCm>zkA-mi97!Q>(9uL>;qOUCHWge5^M!Gz6<?P!du8k0BsAT})&uKK6aS}c=i&{Vo?8gqpAY0l zaJ^JH@LuPk@d{0tSC_B}WNrP6SrOlP2+a))1!dKn+|eUb_q5j-YTawi@rw)z)KSmy z4rDz3d#O&I9)gfQDPqWLHyU)**5~}uA-<};Pv&jz>^<_s-h?rbf{svNXDa5YGV?fe zx&pHIuwxO->^&Cjh>=0qP0$6ORmnzeXLlFZAabc@&^t2IJf+PwK z9J%)=4#0d`z{1lajiJe7%K-#zM_I4}3m~)|WL#iVu+-Nw7@M0;fPZB(_u|m@`X%wBRajQ(WY|U2z;oec?UyU1rnpw&x#r3KDQLGa9y;PUvRW3A)0{oFX|1F zBe=p>JEyZ2OOcjSO~d$}1@NH^Xn1OrYl@y)6^jU=Z0FskNQ$LOGdEpns_@h(W3|OH zCiqWwD_j@Ha;MwZv74vj`3~sa{kGG8LQbu}E_zf#Z7}Iww6ms)p;5&t;}{l8q2NDR zLF}eo>?nT-7AYv@2rW`XVUfbjAX>j=IT4AuNP(;(Sfs$;D0T<=w_j0)64MyuGy?J2 z)0@$CUe!|lXOd)rvcXj2OsMm*Yqwv17Zh7n{CkQPtFxS^M)gWwIwr4jq#^$b$={gUMWyy&vk~OPA^97VV*Z?2DiBaA(wI6fi!Ll( z=d!{4ysr@lZ!g z#1!n=_c~o>{4UeZng2=`Na_6RmS3UE! z-R-J|{xt)KeEE(Ehc2hD*pf_bNY}AM7HDFW)q3Yz#Nmf!;G(B)hgW)p)9ni!?J zILN;{&P;+qxiHYgV0CyZe8Y=ZYSwqep-V@(-qPFa0F7XbJ8Y^x;}~| zLAXQ>NM02{!zk>{0)9R7&BD^m5dz*UOp1Y-bqxg6HPV=RD!XI_{cza;Fl*;}XXdBm z+YEPiZslIiT-GN)(BXb22eW6aE24PNZ8n%aqrM2U zXCws(bM_1b%$||PXM?AIIzI?t_6$jy8XssXqqKO1PD3SZ$VRc=|9PCaU7Gl$o_4qE zyyj_jo<;wt{TtNgX2uNhGSb$4bK+DInGl2%r=Ct;bsC3J&|GxjNvOF(u~!2jWEul7 zDF#N(B>)6W)RD&2KG`KJ=!eS&fFltrtXWuowbq7^6j=Y>`s3#&M#2z3Tgxu|D;!Gx z{2Te_E$i4?IzD$?`O9VG=S;%Cc*BSMy$A8E;?BCnQhXVAOOeVJg}22C!;kRkP+@C~ z!op;jm>DL+RxKC;RxL=1k<2g|5D+Fq8YSfLB`XN3dkoZz?OE~#@p2JvLf7Tt)AzZz zRx_gWHY_iyUQ=aW)?W?WEk)}FX{{d>^x zf!VItD#J5uDnwatZI;)#tu$^5%WLju72FCK1x_C0acFqr#$pv10){6f#Yi+f0Rh7k z(rvK}t6Z{zpyINi-hK&h&c4{#5iy%8o}|NXv{GVWba3bO4H8DVVpTHp-ZAgbYo=T! z7JDS%pNL)NmSA~9-5SL$!C@-S@fxGR5nw#?4E9!V1d^_p zU$TOrhWx>f8XVtLr_}!~Su~hxz5)1I} z0YXOYC!bV2m(6mN_;WsM!|C7+o!Ch(=+!8p0QKqutXHG6T$i6Mrx`4*#UafA*5XK` zL^T}4EFBqxg92c6zQm$aeZ}>{G60U1h1QjA2U0|BKYjohOr@hAbCX$%e>0pUR)z=0=|?UDyFKJ?pWpEvhS z88IcdQ2yR`FLJ+Mt4JO7srg3XxW8z@Z{!qIb23-fwmcY8w z$?EnS396+n|L`DARO7X3i-i&)Ao{W71L;HJl6l@*3%8AT?>nwiDYZpDP4(B@?ym+O zlN&7i1;VujtNy8@2l`%?btbElL>w*=MOb+W?DRlJd2Na3q9G@+iMLk4C05>$v^-iM zL#LC1psc)2H;Dk%<721-{vT7S)Y)Tl|Nj2-Ec+Uyv{Qjx8!YSyjPuH{6P*N=>$=9$>L#~NV)

k4+BB&OI;8GKqtt$$Fg`)YB#WTHJ1_g#Yxj; zLx{Jp!Jv0!WmOHIhbHQocHh5fc0XLd-|gOeq1nZAJ{$W|G{&8c?>m_t9O+&Y#8JU& z6QH1To$`c2I)Yn?@wcWq_LHSZEAJ&nK1zJSIwxRqodSXuel8F)6%xEhf8P=SR#Qa= zVY!>nmMB>B;IrUELaaIg6rKK= z>-}o_Sw`20904HMl1&96QfMg<8ZO%lm~z(~>=LEi`M%(?A;g>J0MvSuepZD~x)Vp7 z);oz?=li`esGKgIP&dhSnq1<4u3J`3` z?gSxHV3rFQC>OirT<%;WxNHdVra1uZtT(S;P!DE%zdys}?%2VW_!IwX6_~un+qwu7 z`=V?PZ+AP)>F&s}n#9e@Mqr`S;khVy1v2ms?Hd_XYl=dYm(~L>UhY9-TRI&C)&{a5 z!~m>p?msK%oDKs!^5qG_Dwrn_mkc33t4S81z3gm>59kp?v8+0KSl7-_q)eq;jdy*l zw|?sMU5R_JkynbmGK}LT>v4i2tIle|ehTm{#YPJB>(u>}NuT|bA}d=`U&B@Ubw=DV zOo$W)zl2YB+yl0=zyvN)_gm*Ha4j1`e7Xrh>*F~bZ0H|JR3nBiub%L)moA%sggf}g ziUcyYuWV^2ts(P-vF&0>!8fqz^x~B$z(e8nw9`~2X5l_z0_DeuubZ z2=UPP0PQ`O%`nzgsQIR=(x}EI-FWS}?KKI*Qz~1FZ&XKheQn+wT1n7QV&Mp4)g_42 zqo^buauAN61U%G)K1G7ek7^>66>1`i3IeDJ5Mlsb0<$K7g_=Ov)fv?RONI~+H33i& zF^=MSC5aLKB{gw6IY5DgM`qwjky+-Ga=?>fccBkYiVR@oFrO4ycv6H_FpnqxxnYpQ zlY%W(Eq>3*Gy0vY{@k88s|}fe%s`K#U}l*;iXVCudRpjF$RPGS%pL_S^eE;nUK}D5_hARnd*g%!BC7W3|+!%A%0Q(Hr z9Of-JTl{~0Vsg_Qu$8RB-|~u(rbv_Yb-8};a9yLro#3;Fow_7A9L^9pNyH$6F8&U! zo=)k-36Oz@T0BVIL=j-tA`xow4h;lQiy*`RK?G(k0t>Z>unOjp#3e(Bhgt;aY6+sI z?2u4j?Z<73DrE(xAFdH{mGFOXy;QOr4%vh5fowCo2LZYV(~6|vQh~ZlFAA{GJrH(v zMpXd1a|9)GLiYe$B8h~=G5;y$QkNS7h8~US27OoGd6mB5mu@|LEBy4?Z~LSpPZ=m% z?5j7Gw(e*eq5q=#P_)QmVH-&mD?3?6>Ax7@75QGWe`P^l@fQQUqF5LRa$i<}5CBvk zrdO;47Q7QvsQM2m?ckH;WW6Gf4 zxv@9n21Ivz(yB1dUBnc^A^N5OiU58y8F*-fhwx44&8QI~IH3^^(Ln%>0749~i^yyQ zV4)EZR>Ayc@RA|KLn8pRcZgV#YHsa2yxYa)-McRW4c9ggj1jteZt1YDC#;}{QFl=S zC?p*kX&3O|jMtM_gprUl-ar6nyo&+?aK<3S04wNBXACSjV}#|7KPkRs2=Pyp!vV^+ zk?`I@{G;02tF>f-si)GLGh-jmYW7k7`T#MphhS!5o&C<`(TC4&R?#(~SKVwXza3)P zxVQ6-AdN<)QYd6Hi6lZI@Z#`v8WkFxz{*M>;8BB9u?8oRK#=>A073vzDa-~Z0$W$Z ziLfs|9$Pkqc>5X*>PP96#`R{k7jJ&)Lii!|Y4DzE-FL#i!g4{-YmLE$HJgt6`-tY;61V$r7p59164 ze~qTkESevR-c48!f-TumAVdnxq5%U%W0&+9&T4|oh7fO>1JLA+9G-7+ag}PDBiZ_{ zbWfj;{&fE8x_x@?PTKPLBITezT~k1Zd>MzjFueDuJEQkrj|lHQv*St%F7@%0CAR=9 zy!QyZTCPSHy(|byUIp(x*y0|jERX$u%kDwgx6gMUiAZ}ioQr$VT4I|Pc4TxgJWwN# z_0L>}qC<|tp*ar}kLnf`&&~$LGh0LQ|0|a1m7t!Xc!XV@QA>_I8{ejZa$Jy062xfH4(v5PNgcTf3g5yCNCh^M}|dgX#lyx`$Y&LlpvZ z@8xG(ntKBaqYT2nxNov-2=Vqc*qC9nMQD?ddYWoFB`%R(*l6YBZLjU56)P-8b8cypEp!@Zz+`Wv92NU1hE~igXPv8IxM%8$RL2_76>swJ%hR20v48A2pckxle}aI@l7{_0lK4P z@2>(C&gz3wZE{0TZbjIMa;iNGzpFyho|EeO!FG_gi7G(hBoo8dkaT&HaffJ|u$m|A z!l&R+&2PeLo)`uKsCf_qfGV(5^T0yQBkYU&$CeEt9%>$-i2|L{3q2863jVR)u`u?z zb#ii2rEy~Lhu@9+KdN;fSD4CqCjLuXw>E%SL9n4`QIAosP)K`lTtUFYKr0I$zvN=B zda!}^7<_I_5C&QhVgQ?^ffiU8Xc3mX`BeFmA;iO~2cQul#EfW@M74KEZ&woRdHx@< zhs?TxhAcsSBde&sZLs=A`cU6XMP)73H?UCO2peLbV~y?^L4$};-(ZXV6ulz>67%Cs zN}oS;Iy`85MPg$_ZG@3&X-`p|i2M^X(xcMFKIcXUDM|BfzV?fCZ-b;#z?P+C_cmgbRxj<)8Y-)*YLo7_9V7%XP?2QGYzxA!Hd- zQk>s(*Z@w( zQ+MC)#HX6gsSD>E_y5<{zYufb&Mjre^=jD!A@=~aPxnTT$S+pxlkSMoc6e&L<*pph zz|)Z2z0XH9TTUoOcU`~oLLxn(bjv;!fxc%JYQ92myFLkc6|^~;$CLH-i;AA>ZK?bH z==Oq6!o=vQn7HQfhOJFMr8i%(%&z{of3*CToyjIW`-^8+waO*k(6`&J94h5eDdUw? z^R-XzD!VAV_AO7xQ~Ylu@Pp|R1tThRIh&))!!=qfI4wj4zIR%06gqi_^;xQ4t6}V< zw(@A(9go=I8|*Q;zjk~)d-m&%0o>rEsY%hZfv3}2J2{HPVnts)`KX_@=H1)Me{MgZ zf7|ip(MNA8pL)#5>33&kjtT#(r{Z2z)u3tA&Sx(B#n?ZzcS_OFw22@0>Y{bo;BxdEVl8o2tduZ3X-8CoW@WjzmiLaki z)dvq2DxT}EjlcS+(qcBv!p55XtT$IHzx{Upv_9zW0CmUQ&H zf_ziuPEHjAo4F%1xe2xS!7}fb8?+zkY3%kFQ{-a|{~2^yr@gAGB*@aLFRQ~hLuXsv zcuP9R=ulUr`b72B1jp<=@3-FWe7Q}p9^IQ3&ej)nnSJuOUAiteL+s!oa?Ka(`dY%L zTMI4;Puuo3A8eOhoW{A9KkHHGZhGQk@t>Z2YlKRy@P7yam|^ElV`sf3GJ-DEW__Mdmx zZp(<)tvj;TAYN7`)CJEPZd~H=Q02@(gQJ`uk6PQw`o4^S10L(#3gdNB=Qh*a=D5l| zWrog0JUE;;nvfm5>-b!qiDQvV!^^(FgoiFW`|d29G{d_O@QaGFHhh-k9joLjI5?8@ z-ls+4;zij{ahw|4C&V1IXHq}Aq@^kLKecFe%)XL4ZF1pF>e2mPb-SFBJcU31R~N2X zN_`aa(A`i^<@e^5-Vq-8ckiCKS>HI+WT3X&?bEwGHb?G`O+Z4M=|#wPMC zPmiZvV&}PdjJ?-$SH5auRn1sIgR8l>n40?YjcYtTTLi|A)UFUdII!^j*yR+h$ELjm zOR074``iB8qEc8NUAH#D@X_I%FuTKjUtHW8_H0jXZ<2egwXsOdt9SOAG`na;?!DVR zZ_>?rTfh8_aPryLopx2pZ@jzC$^Yj3P)%g!Z;z*C-;cF*8vGF3GqJbSf8poHx)_g+ z#qX183wk$NBBx(`R$26%9-sF29{)Ka-1dHht8n3yk3rSLjwi1uG)ATczDP~__W zP7hle{vLTDwr5tYIz0IJSme=Rsj}jW3saf?zLVW8+6L$H3c`6yP7K$+QE}=pY@Z*S zoeb0ZCa=BlP|rl!r&Z;o$IYVt&*@SfhFZ-|^$pEEF^e<9N42ZfdfBXJjf8}R;@xc? z)#*)?`}<7J#`=sb&dv;Vtkqg5tFm)wE6@|m@M(N{vDtE+RdfI31;q!3)^*imB7Vif z^Zr-*S2Uj$9=#j+b%)utC*K+sEZ6?)8v4CX<>G^JtyiWWBo4&*-&|YtGJG+;bwhTB z(Mqbdij>9Bw`pVJ0HSwjhQO7nLs^DjcRZ|poPPC?;-UM`jA@oX9jB&ZxyQ;=KSz_U+;Ro&2*1~u(R_b5XYJC&mjAn!v@o2b?Deawk znt1bFQNxW_bf;gF`rAjY?2WW#sf(Tf(y&zY4O?k>^M+Q2%CwinoN?BhQr_*YNr8q_ zhFNdqN`&Y$wL9YMr(*m{^^_&s9wjbv*)|&*-o7>eYWk;_K#%;CzfQ=_pd0)ArX&(u zLOUwBf8CkOWN0xo=62a=Ce+lNvu=>9wzB%PSo~pezF_vmWVMHF+69TR#@^~%DXRZ6 z?%2n+K7ZtEb#O?mYO3T@)BDqBpNdY2bc9d(tl9i=ppLP1-A}X4md0++N5vki8{6{* z2VbDiE8U!~{p$A2TJBoX9sryU@hP&PybKF&GNX`G)#OtT>STUyZ%bBP9a_97ZTUuJq z)hV03H92JCs`l1l*Ub3#>iJOD$V)}(&&{fayCt66Tl^zfQLVoJ>fXr0{~p`RrDV!w z44I6Lecj4b;nfO`rJ<$qfTuzW3dh0TAz1Tu?&n~l_T|7$~xyuXGRKkRP|X* zPAsf%Z;9o0%~9MbpVlT&KdB@$=M%$!QFxQn@YM#z&f}fl^;buaPxC!6wJERamozLt zRa6|+_OLF!;3Oq&=*Nk332`*{z3HAqqJ9~c8#=!~zAODrskxx}?$rW+n#+nFHbeBqr{K{D zLD7$a`B~jp&K|a3+ac+AG3{1=)$EtT5<>j=1+DQTYG#p~MVG&xTKlhjbZ#A;XHB7c z=GpgdZ@fBg{2M&fR2z8ae3rfGCW*1zIl0`Uzkes1t<3V8d)5>7FSo|22BPakI?tio z@=qr}GE_v9Lei$5@Emb}&?LKO+*aJGQ!UZE@KhOblKWA}uOr#_P2BUWtQEtIAM}Ur zTeVu~&EPAYqT`(xFGuRDj(lL~Nry|AbJ@?URq>>N7kn*eu7MoclcF zw9o5Ci{H(6KXumaKHw2t^i5%)z}M)ycS+X!60f1e2d8$7OSyYCotBPCDc4}BUc15 z$jryddB|^Tx6P_abF1Aw?P)PB!HYX&|MTDafFNW+p3k}+qEav+(QPV}w=Z%bEJ38W zMET=f$y=wB|?5}x74R|b}G}=vdUJ4Lft$XT7$xFvAt=B9cqorKD_gY^fN3iy`Jxp zxNzR(qVlzJ`raYw(W*BeE5E;-3aZ@o`1QvmOP>oS`$hd1?b3eL`e@924VRZvsS;ak wzEq=BFh96$T>QLz&Ahvdvf`OyX@}OQNk+f>2f$BR`A*5d+@fn|ahihrKkfG5(*OVf literal 0 HcmV?d00001 diff --git a/openspec/changes/fix-lab3-postgres-pool-recovery/design.md b/openspec/changes/fix-lab3-postgres-pool-recovery/design.md index 99e55e24..c7e2200f 100644 --- a/openspec/changes/fix-lab3-postgres-pool-recovery/design.md +++ b/openspec/changes/fix-lab3-postgres-pool-recovery/design.md @@ -20,6 +20,12 @@ - Дополнительное обращение к PostgreSQL → стоимость измеряется в следующем этапе трека, старые RPS не объявляются результатом SQLAlchemy. - Разрыв после успешной проверки всё ещё возможен → существующие ограниченные `503`, сообщение повторяется клиентом с тем же ключом. +## Изоляция собственного теста SIGTERM + +Полный набор при работающем LB завис на `test_drain.py`: встроенный gunicorn тоже пытался слушать 18083, а тест принимал readiness LB за готовность дочернего процесса. Прогон прерван после 46 успешных тестов, вывод сохранён в `evidence/grade3-student-tests-interrupted.txt`. Тест выбирает свободный порт 23000–23999 через существующую fixture-утилиту и ограничивает ожидание закрытия listener. Поведение приложения не меняется. + +Следующий полный прогон выявил стартовую гонку тестовой БД: SQLAlchemy пробрасывает `CannotConnectNowError` с SQLSTATE `57P03` при открытии соединения до готовности PostgreSQL. Fixture миграций использует существующий классификатор `unavailable`, который учитывает SQLSTATE, и повторяет только временные отказы до прежнего дедлайна 60 с. Ошибки схемы не скрываются. Исходный вывод (70 passed / 201 errors) сохранён в `evidence/grade3-student-tests-startup.txt.gz`. + ## Migration Plan Миграция схемы не нужна. Собрать текущий API, переснять публичные smoke/persistence/failover обеих реплик, гонки ключа и потерянный ответ, `make test-faults`, собственные тесты. Сохранить исходное падение отдельно. Откат — предыдущий образ на той же схеме и volumes, с известной регрессией пула. diff --git a/openspec/changes/fix-lab3-postgres-pool-recovery/proposal.md b/openspec/changes/fix-lab3-postgres-pool-recovery/proposal.md index ddfcf758..42937fa0 100644 --- a/openspec/changes/fix-lab3-postgres-pool-recovery/proposal.md +++ b/openspec/changes/fix-lab3-postgres-pool-recovery/proposal.md @@ -6,6 +6,7 @@ - Проверять соединение пула перед началом операции и заменять закрытое; не повторять уже начатые транзакции и неидемпотентные POST. - Добавить регрессионную проверку разрыва простаивающего соединения и усилить проверку записей после возвращения PostgreSQL через LB. +- Исправить выявленные проблемы запуска собственных тестов: коллизию порта gunicorn с LB и стартовую гонку тестовой БД. - Переснять уровень «3» и обновить строки «Перенос lab2: отказы PostgreSQL» и «3.5» в `REPORT.md`, сохранив неуспешный прогон. ## Capabilities diff --git a/openspec/changes/fix-lab3-postgres-pool-recovery/tasks.md b/openspec/changes/fix-lab3-postgres-pool-recovery/tasks.md index df6b5acf..9dd0833f 100644 --- a/openspec/changes/fix-lab3-postgres-pool-recovery/tasks.md +++ b/openspec/changes/fix-lab3-postgres-pool-recovery/tasks.md @@ -1,7 +1,9 @@ ## 1. Регрессия и исправление платформы -- [ ] 1.1 Воспроизвести первый `503` после закрытия простаивающего соединения отдельным тестом `test_pool.py`, сохранить исходное падение -- [ ] 1.2 Включить проверку соединения до выдачи из пула; целевой тест и `make student-test`, `make lint`, `make check` проходят +- [x] 1.1 Воспроизвести первый `503` после закрытия простаивающего соединения отдельным тестом `test_pool.py`, сохранить исходное падение +- [x] 1.2 Включить проверку соединения до выдачи из пула; целевой тест и `make student-test`, `make lint`, `make check` проходят +- [ ] 1.3 Изолировать порт gunicorn в `test_drain.py` от LB и ограничить ожидание закрытия listener; тест и полный набор проходят при работающем стенде +- [x] 1.4 Дождаться готовности тестового PostgreSQL при SQLSTATE 57P03 через существующий классификатор временных отказов; полный набор проходит ## 2. Реальный стенд уровня «3» diff --git a/src/messenger/shared/infrastructure/postgres.py b/src/messenger/shared/infrastructure/postgres.py index 5b3531e2..b2d5f550 100644 --- a/src/messenger/shared/infrastructure/postgres.py +++ b/src/messenger/shared/infrastructure/postgres.py @@ -56,6 +56,7 @@ class Database: max_overflow=0, pool_timeout=acquire_timeout, pool_recycle=60, + pool_pre_ping=True, hide_parameters=True, connect_args={ "timeout": connect_timeout, diff --git a/student-tests/api/test_drain.py b/student-tests/api/test_drain.py index fe5fccc1..58462672 100644 --- a/student-tests/api/test_drain.py +++ b/student-tests/api/test_drain.py @@ -9,7 +9,7 @@ from pathlib import Path import httpx -from conftest import DATABASE, new_user +from conftest import DATABASE, free_port, new_user from messenger.shared.infrastructure.readiness import DRAINING DRAIN = 1.0 @@ -67,7 +67,7 @@ def begin_registration(port: int) -> tuple[socket.socket, bytes]: def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests(tmp_path: Path) -> None: - port = 18083 + port = free_port() settings = { "HOST": "127.0.0.1", "PORT": str(port), @@ -98,6 +98,7 @@ def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests(tmp_path time.sleep(0.3) during = status(port, "/health/ready"), status(port, "/health/live") while status(port, "/health/live") == 200: + assert time.monotonic() - sent < SHUTDOWN + 2, log_path.read_text() time.sleep(0.05) closed = time.monotonic() - sent pending.sendall(rest) diff --git a/student-tests/api/test_pool.py b/student-tests/api/test_pool.py index 6f64c661..3ccb5881 100644 --- a/student-tests/api/test_pool.py +++ b/student-tests/api/test_pool.py @@ -7,11 +7,21 @@ import pytest from sqlalchemy import func, select from sqlalchemy.exc import DBAPIError, IntegrityError -from conftest import actor, keyed, make_chat, register, resolve, running_app +from conftest import ( + actor, + keyed, + make_chat, + new_user, + register, + resolve, + running_app, + sql_connection, +) from messenger.identity.domain.models import Session from messenger.identity.infrastructure.postgres import ( PostgresIdentityUnitOfWork, PostgresSessionDirectory, + UserRow, ) from messenger.messaging.domain.models import Message from messenger.messaging.domain.values import MessageText @@ -65,6 +75,28 @@ async def test_idle_session_does_not_acquire_connection() -> None: await database.ping() +async def test_closed_idle_connection_is_replaced_before_registration() -> None: + async with running_app(db_pool_max_size=1) as http: + database = await resolve(http, Database) + async with database.session() as session: + pid = await session.scalar(select(func.pg_backend_pid())) + async with sql_connection() as connection: + assert await connection.scalar(select(func.pg_terminate_backend(pid, 1000))) + body = new_user("reconnect") + response = await http.post("/api/v1/users", json=body) + assert response.status_code == 201 + assert response.headers["X-Instance-Id"] and response.headers["X-Request-Id"] + async with database.session() as session: + assert ( + await session.scalar( + select(func.count()) + .select_from(UserRow) + .where(UserRow.username == body["username"]) + ) + == 1 + ) + + async def test_readiness_timeout_releases_waiter() -> None: async with running_app(db_pool_max_size=1, db_acquire_timeout_seconds=2) as http: database = await resolve(http, Database) diff --git a/student-tests/conftest.py b/student-tests/conftest.py index 2c6f659a..0d6f7171 100644 --- a/student-tests/conftest.py +++ b/student-tests/conftest.py @@ -16,7 +16,6 @@ from alembic.config import Config from fastapi import FastAPI from httpx import ASGITransport, AsyncClient from sqlalchemy import make_url -from sqlalchemy.exc import DBAPIError from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine from sqlalchemy.pool import NullPool @@ -25,6 +24,7 @@ from messenger.main import create_app from messenger.messaging.config import MessagingSettings from messenger.shared.config import PlatformSettings from messenger.shared.entrypoints.config import TransportSettings +from messenger.shared.infrastructure.postgres import unavailable ROOT = Path(__file__).resolve().parents[1] POSTGRES_IMAGE = re.compile(r"image: (postgres:\S+)") @@ -57,8 +57,8 @@ async def migrated(dsn: str) -> None: try: await upgrade(dsn) return - except OSError, DBAPIError: - if time.monotonic() > deadline: + except Exception as error: + if not unavailable(error) or time.monotonic() > deadline: raise await asyncio.sleep(0.5) diff --git a/student-tests/stack/test_faults.py b/student-tests/stack/test_faults.py index 6f57a9c2..5a67c497 100644 --- a/student-tests/stack/test_faults.py +++ b/student-tests/stack/test_faults.py @@ -211,6 +211,13 @@ async def test_postgres_stopped_gives_bounded_503_and_recovers(http: AsyncClient me = await http.get("/api/v1/users/me", headers=bearer(token)) print(f"postgres запущен: готовность через {recovered:.1f} с, прежний токен {me.status_code}") assert me.json() == user + instances = set() + for _ in range(40): + response = await http.post("/api/v1/users", json=new_user("recovered")) + assert response.status_code == 201 + instances.add(response.headers["X-Instance-Id"]) + print(f"postgres запущен: 40 регистраций 201 через {len(instances)} реплики без рестарта API") + assert len(instances) >= 2 assert started_at("api") == api_started