From 8105f900976fd422c1760575012ce2503fb1ce4f Mon Sep 17 00:00:00 2001 From: EgorAnti Date: Tue, 3 Nov 2026 20:53:02 +0300 Subject: [PATCH 1/3] =?UTF-8?q?=D0=9B=D0=A0=203:=20=D0=BC=D0=B0=D1=81?= =?UTF-8?q?=D1=88=D1=82=D0=B0=D0=B1=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D0=BD?= =?UTF-8?q?=D0=B8=D0=B5=20=D0=B8=20=D0=BE=D1=82=D0=BA=D0=B0=D0=B7=D0=BE?= =?UTF-8?q?=D1=83=D1=81=D1=82=D0=BE=D0=B9=D1=87=D0=B8=D0=B2=D0=BE=D1=81?= =?UTF-8?q?=D1=82=D1=8C=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 25 +- .gradle/9.7.1/checksums/checksums.lock | Bin 0 -> 17 bytes .../executionHistory/executionHistory.bin | Bin 0 -> 212835 bytes .../executionHistory/executionHistory.lock | Bin 0 -> 17 bytes .gradle/9.7.1/fileChanges/last-build.bin | Bin 0 -> 1 bytes .gradle/9.7.1/fileHashes/fileHashes.bin | Bin 0 -> 25997 bytes .gradle/9.7.1/fileHashes/fileHashes.lock | Bin 0 -> 17 bytes .../9.7.1/fileHashes/resourceHashesCache.bin | Bin 0 -> 22577 bytes .gradle/9.7.1/gc.properties | 0 .../buildOutputCleanup.lock | Bin 0 -> 17 bytes .gradle/buildOutputCleanup/cache.properties | 2 + .gradle/buildOutputCleanup/outputFiles.bin | Bin 0 -> 19055 bytes .gradle/vcs-1/gc.properties | 0 Dockerfile | 37 ++ Makefile | 22 +- README.md | 113 +++++ REPORT.md | 440 +++++++++++++++++- build.gradle | 55 +++ compose.yaml | 122 +++++ evidence/failover-each-2026-11-03.json | 24 + evidence/level4-2026-11-03.md | 49 ++ evidence/node-failover-2026-11-03.json | 42 ++ evidence/node-failover-2026-11-03.md | 33 ++ evidence/rolling-update-2026-11-03.json | 23 + gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 47505 bytes gradle/wrapper/gradle-wrapper.properties | 9 + gradlew | 248 ++++++++++ gradlew.bat | 82 ++++ infra/README.md | 47 ++ infra/nginx.conf | 70 +++ scripts/restart.cmd | 8 + scripts/restart.example.sh | 0 scripts/restart.sh | 13 + scripts/restore-api.cmd | 6 + scripts/restore-api.sh | 9 + scripts/rolling-update.cmd | 16 + scripts/rolling-update.sh | 22 + scripts/stop-one.cmd | 25 + scripts/stop-one.example.sh | 0 scripts/stop-one.sh | 28 ++ scripts/swarm-lab-down.ps1 | 19 + scripts/swarm-lab-up.ps1 | 138 ++++++ settings.gradle | 1 + .../MtusiMessengerApplication.java | 18 + .../mtusi_messenger/cache/UserCache.java | 83 ++++ .../mtusi_messenger/config/JacksonConfig.java | 28 ++ .../mtusi_messenger/config/OpenApiConfig.java | 21 + .../config/PasswordConfig.java | 14 + .../config/SecurityConfig.java | 92 ++++ .../mtusi_messenger/config/WebConfig.java | 19 + .../controller/ChatController.java | 83 ++++ .../controller/HealthController.java | 58 +++ .../controller/MessageController.java | 79 ++++ .../controller/SessionController.java | 41 ++ .../controller/UserController.java | 32 ++ .../mtusi_messenger/dto/ChatPageResponse.java | 8 + .../mtusi_messenger/dto/ChatResponse.java | 18 + .../dto/CreateChatRequest.java | 19 + .../dto/CreateMessageRequest.java | 8 + .../dto/CreateUserRequest.java | 20 + .../mtusi_messenger/dto/ErrorResponse.java | 8 + .../mtusi_messenger/dto/HealthResponse.java | 3 + .../dto/MessagePageResponse.java | 8 + .../mtusi_messenger/dto/MessageResponse.java | 19 + .../mtusi_messenger/dto/SessionResponse.java | 10 + .../mtusi_messenger/dto/UserResponse.java | 18 + .../exception/ApiException.java | 14 + .../exception/ApiExceptions.java | 14 + .../example/mtusi_messenger/model/Chat.java | 11 + .../model/IdempotentMessage.java | 3 + .../mtusi_messenger/model/Message.java | 6 + .../mtusi_messenger/model/MessageKey.java | 35 ++ .../mtusi_messenger/model/Session.java | 15 + .../example/mtusi_messenger/model/User.java | 6 + .../repository/ChatRepository.java | 14 + .../repository/MessageRepository.java | 18 + .../repository/PostgresChatRepository.java | 106 +++++ .../repository/PostgresMessageRepository.java | 134 ++++++ .../repository/PostgresSessionRepository.java | 51 ++ .../repository/PostgresUserRepository.java | 72 +++ .../repository/SessionRepository.java | 12 + .../repository/UserRepository.java | 14 + .../security/ApiAuthenticationFilter.java | 130 ++++++ .../DatabaseAuthenticationProvider.java | 63 +++ .../RestAuthenticationEntryPoint.java | 46 ++ .../security/SessionPrincipal.java | 14 + .../service/AuthenticationService.java | 104 +++++ .../mtusi_messenger/service/ChatService.java | 38 ++ .../service/CursorService.java | 40 ++ .../service/MessageService.java | 89 ++++ .../mtusi_messenger/service/UserService.java | 34 ++ .../service/UuidValidator.java | 27 ++ .../web/GlobalExceptionHandler.java | 48 ++ .../mtusi_messenger/web/InstanceIdFilter.java | 47 ++ .../mtusi_messenger/web/NotUnicodeBlank.java | 21 + .../mtusi_messenger/web/RequestIdFilter.java | 40 ++ .../web/UnicodeBlankValidator.java | 23 + .../mtusi_messenger/web/UnicodeSize.java | 35 ++ .../web/UnicodeSizeValidator.java | 32 ++ src/main/resources/application.yml | 62 +++ .../migration/V1__create_messenger_schema.sql | 60 +++ .../V2__add_credentials_and_sessions.sql | 28 ++ .../migration/V3__add_message_idempotency.sql | 25 + .../MtusiMessengerApplicationTests.java | 368 +++++++++++++++ .../mtusi_messenger/cache/UserCacheTest.java | 31 ++ student_tests/README.md | 11 + student_tests/failover_each.py | 135 ++++++ student_tests/idempotency_concurrent.py | 53 +++ student_tests/node_failover.py | 182 ++++++++ student_tests/rolling_update.py | 110 +++++ swarm/nginx.Dockerfile | 3 + swarm/nginx.conf | 59 +++ swarm/stack.app.yaml | 75 +++ swarm/stack.state.yaml | 60 +++ tests/resilience.py | 3 +- 115 files changed, 5018 insertions(+), 38 deletions(-) create mode 100644 .gradle/9.7.1/checksums/checksums.lock create mode 100644 .gradle/9.7.1/executionHistory/executionHistory.bin create mode 100644 .gradle/9.7.1/executionHistory/executionHistory.lock create mode 100644 .gradle/9.7.1/fileChanges/last-build.bin create mode 100644 .gradle/9.7.1/fileHashes/fileHashes.bin create mode 100644 .gradle/9.7.1/fileHashes/fileHashes.lock create mode 100644 .gradle/9.7.1/fileHashes/resourceHashesCache.bin create mode 100644 .gradle/9.7.1/gc.properties create mode 100644 .gradle/buildOutputCleanup/buildOutputCleanup.lock create mode 100644 .gradle/buildOutputCleanup/cache.properties create mode 100644 .gradle/buildOutputCleanup/outputFiles.bin create mode 100644 .gradle/vcs-1/gc.properties create mode 100644 Dockerfile create mode 100644 build.gradle create mode 100644 compose.yaml create mode 100644 evidence/failover-each-2026-11-03.json create mode 100644 evidence/level4-2026-11-03.md create mode 100644 evidence/node-failover-2026-11-03.json create mode 100644 evidence/node-failover-2026-11-03.md create mode 100644 evidence/rolling-update-2026-11-03.json create mode 100644 gradle/wrapper/gradle-wrapper.jar create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100644 gradlew create mode 100644 gradlew.bat create mode 100644 infra/nginx.conf create mode 100644 scripts/restart.cmd mode change 100755 => 100644 scripts/restart.example.sh create mode 100644 scripts/restart.sh create mode 100644 scripts/restore-api.cmd create mode 100644 scripts/restore-api.sh create mode 100644 scripts/rolling-update.cmd create mode 100644 scripts/rolling-update.sh create mode 100644 scripts/stop-one.cmd mode change 100755 => 100644 scripts/stop-one.example.sh create mode 100644 scripts/stop-one.sh create mode 100644 scripts/swarm-lab-down.ps1 create mode 100644 scripts/swarm-lab-up.ps1 create mode 100644 settings.gradle create mode 100644 src/main/java/com/example/mtusi_messenger/MtusiMessengerApplication.java create mode 100644 src/main/java/com/example/mtusi_messenger/cache/UserCache.java create mode 100644 src/main/java/com/example/mtusi_messenger/config/JacksonConfig.java create mode 100644 src/main/java/com/example/mtusi_messenger/config/OpenApiConfig.java create mode 100644 src/main/java/com/example/mtusi_messenger/config/PasswordConfig.java create mode 100644 src/main/java/com/example/mtusi_messenger/config/SecurityConfig.java create mode 100644 src/main/java/com/example/mtusi_messenger/config/WebConfig.java create mode 100644 src/main/java/com/example/mtusi_messenger/controller/ChatController.java create mode 100644 src/main/java/com/example/mtusi_messenger/controller/HealthController.java create mode 100644 src/main/java/com/example/mtusi_messenger/controller/MessageController.java create mode 100644 src/main/java/com/example/mtusi_messenger/controller/SessionController.java create mode 100644 src/main/java/com/example/mtusi_messenger/controller/UserController.java create mode 100644 src/main/java/com/example/mtusi_messenger/dto/ChatPageResponse.java create mode 100644 src/main/java/com/example/mtusi_messenger/dto/ChatResponse.java create mode 100644 src/main/java/com/example/mtusi_messenger/dto/CreateChatRequest.java create mode 100644 src/main/java/com/example/mtusi_messenger/dto/CreateMessageRequest.java create mode 100644 src/main/java/com/example/mtusi_messenger/dto/CreateUserRequest.java create mode 100644 src/main/java/com/example/mtusi_messenger/dto/ErrorResponse.java create mode 100644 src/main/java/com/example/mtusi_messenger/dto/HealthResponse.java create mode 100644 src/main/java/com/example/mtusi_messenger/dto/MessagePageResponse.java create mode 100644 src/main/java/com/example/mtusi_messenger/dto/MessageResponse.java create mode 100644 src/main/java/com/example/mtusi_messenger/dto/SessionResponse.java create mode 100644 src/main/java/com/example/mtusi_messenger/dto/UserResponse.java create mode 100644 src/main/java/com/example/mtusi_messenger/exception/ApiException.java create mode 100644 src/main/java/com/example/mtusi_messenger/exception/ApiExceptions.java create mode 100644 src/main/java/com/example/mtusi_messenger/model/Chat.java create mode 100644 src/main/java/com/example/mtusi_messenger/model/IdempotentMessage.java create mode 100644 src/main/java/com/example/mtusi_messenger/model/Message.java create mode 100644 src/main/java/com/example/mtusi_messenger/model/MessageKey.java create mode 100644 src/main/java/com/example/mtusi_messenger/model/Session.java create mode 100644 src/main/java/com/example/mtusi_messenger/model/User.java create mode 100644 src/main/java/com/example/mtusi_messenger/repository/ChatRepository.java create mode 100644 src/main/java/com/example/mtusi_messenger/repository/MessageRepository.java create mode 100644 src/main/java/com/example/mtusi_messenger/repository/PostgresChatRepository.java create mode 100644 src/main/java/com/example/mtusi_messenger/repository/PostgresMessageRepository.java create mode 100644 src/main/java/com/example/mtusi_messenger/repository/PostgresSessionRepository.java create mode 100644 src/main/java/com/example/mtusi_messenger/repository/PostgresUserRepository.java create mode 100644 src/main/java/com/example/mtusi_messenger/repository/SessionRepository.java create mode 100644 src/main/java/com/example/mtusi_messenger/repository/UserRepository.java create mode 100644 src/main/java/com/example/mtusi_messenger/security/ApiAuthenticationFilter.java create mode 100644 src/main/java/com/example/mtusi_messenger/security/DatabaseAuthenticationProvider.java create mode 100644 src/main/java/com/example/mtusi_messenger/security/RestAuthenticationEntryPoint.java create mode 100644 src/main/java/com/example/mtusi_messenger/security/SessionPrincipal.java create mode 100644 src/main/java/com/example/mtusi_messenger/service/AuthenticationService.java create mode 100644 src/main/java/com/example/mtusi_messenger/service/ChatService.java create mode 100644 src/main/java/com/example/mtusi_messenger/service/CursorService.java create mode 100644 src/main/java/com/example/mtusi_messenger/service/MessageService.java create mode 100644 src/main/java/com/example/mtusi_messenger/service/UserService.java create mode 100644 src/main/java/com/example/mtusi_messenger/service/UuidValidator.java create mode 100644 src/main/java/com/example/mtusi_messenger/web/GlobalExceptionHandler.java create mode 100644 src/main/java/com/example/mtusi_messenger/web/InstanceIdFilter.java create mode 100644 src/main/java/com/example/mtusi_messenger/web/NotUnicodeBlank.java create mode 100644 src/main/java/com/example/mtusi_messenger/web/RequestIdFilter.java create mode 100644 src/main/java/com/example/mtusi_messenger/web/UnicodeBlankValidator.java create mode 100644 src/main/java/com/example/mtusi_messenger/web/UnicodeSize.java create mode 100644 src/main/java/com/example/mtusi_messenger/web/UnicodeSizeValidator.java create mode 100644 src/main/resources/application.yml create mode 100644 src/main/resources/db/migration/V1__create_messenger_schema.sql create mode 100644 src/main/resources/db/migration/V2__add_credentials_and_sessions.sql create mode 100644 src/main/resources/db/migration/V3__add_message_idempotency.sql create mode 100644 src/test/java/com/example/mtusi_messenger/MtusiMessengerApplicationTests.java create mode 100644 src/test/java/com/example/mtusi_messenger/cache/UserCacheTest.java create mode 100644 student_tests/README.md create mode 100644 student_tests/failover_each.py create mode 100644 student_tests/idempotency_concurrent.py create mode 100644 student_tests/node_failover.py create mode 100644 student_tests/rolling_update.py create mode 100644 swarm/nginx.Dockerfile create mode 100644 swarm/nginx.conf create mode 100644 swarm/stack.app.yaml create mode 100644 swarm/stack.state.yaml diff --git a/.env.example b/.env.example index d32b9be0..a260ab09 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,18 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 +# Copy to .env for the local synthetic lab environment. Never commit .env. APP_PORT=8080 -APP_ENV=lab +DATABASE_NAME=messenger +DATABASE_USER=messenger +DATABASE_PASSWORD=replace-with-a-local-random-password +DATABASE_POOL_SIZE=10 +DATABASE_MIN_IDLE=2 +DATABASE_QUERY_TIMEOUT_SECONDS=5 +DATABASE_TRANSACTION_TIMEOUT_SECONDS=5 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 +USER_CACHE_TTL_SECONDS=300 IDEMPOTENCY_TTL_SECONDS=86400 +REDIS_CONNECT_TIMEOUT=1s +REDIS_COMMAND_TIMEOUT=500ms +REDIS_POOL_MAX_ACTIVE=8 +REDIS_POOL_MAX_IDLE=8 +REDIS_POOL_MIN_IDLE=0 +REDIS_POOL_MAX_WAIT=500ms diff --git a/.gradle/9.7.1/checksums/checksums.lock b/.gradle/9.7.1/checksums/checksums.lock new file mode 100644 index 0000000000000000000000000000000000000000..83712aa7b6713b9a96af925996bd602bd7b7c51a GIT binary patch literal 17 TcmZQxDyJ3@%6jDs0~7!NCy4`J literal 0 HcmV?d00001 diff --git a/.gradle/9.7.1/executionHistory/executionHistory.bin b/.gradle/9.7.1/executionHistory/executionHistory.bin new file mode 100644 index 0000000000000000000000000000000000000000..1c97ae403d6eda98e421d66b8921823acaec9b6b GIT binary patch literal 212835 zcmeEv2Ygdy_i&Q3=Vea;l?@4Z<=z_vrBXHs$P^-YX9=WDNzy_=_TGE%y=3pbWXe!B z$R09e%a9>_=ic0;NgL7{llQORudlDRN%Ne0&Uwyr_Hz=Y(G*7BtN+s^{jWgMuj#*~ zb0D1q=^RMsKspD~IgrkQbPl9*Ae{s097yLtItS7@kj{a04y1D+odf9{NasL02hut4 z{|g6N03q32#1yWqlYhzGMWaD;z%LmVj+|ZD_~7ZU%Ad8B-@5AuTW02M`@YP=@visF zc6Pt7`|ifmDT6X(RNmLrjdZ`?q{&ok_MtbnEAQ(JIoePZ$HsA8x=NJQD3IMO-8xvhSrekWS*>g$QaW;RnTFjH$JMx3^N(Dv?W#n*hKj? zl6P8cv85vg!6N^{S{sQ@R$!fMLmSV^?>K7OVkJv-yj`>->}hV%V)LqJAJ^{HVN{+? zT|$C(@ev)9$g}6s?DS$q$AlPz(`K{s9azBH66+N0fTAHvl-PKyv$^{(Ui`7AV9P{r z@P-(bygZs!sXs@C>v;(BB>KXsU8^^D*?4T(-KgDC>_WeQ4OTWbI-ZRdTZ(oE@UMV2 z#lrSKJkVrx*frHs$3Uf2Mx>!H+{7NoznPzDk6owH*3)E8a6PPR97uGbIsOroWvX6m zT)fkfN3%ZlNBa=3tV==2s?;82u*Ae!#TYTxsmM~q=marVu-WyFIFYwV7JhZA0RHc} zxj0oET|`9;V`CFbxqcrmuTcdc%QP=u%V)cO(%*))to3!XdX2woj}yaUY(l)1wOgEt zc{H26mUyIIJ`AcnH1d458F11|CnT5AX-QDw_(-B%St&MQogK7;ZE?y0&!gGk^|+MU zN7X!yU7z-PSaJn4K?Ug6TlPqMv?GT$Vz@qzm9-I^UdKrHX9SUpkJfsYC@z!uGG>zRq3_q#g9Sosh~S?mtY zc=y{9Yq2(tt!1~{?Aj5YH$m0vl0E{j8oSvWSh?3u`e*>&&hmba3fUlb75R84%UMOu zbbo&biIKM;yN45&Sl;eBj@C#Cwxc#s0-9axSuM-0OGFxh#;s{(9S+S@H|V54s-mqK z?f#Qpw2G`l)Qor7;_bXBiwvB2&SC{|j_8LIN-P5)*gYBc-NGll%0&Zkc4#y>$rg)m? zQJ%*bmSHgXKSq!!{Eb5~EC3|E0?9>U^(Kp7Lx@hrw0Ns2-VhTx1e>6fA;XaT8u5$-+eC~)o_ zqA;V0B}fS)DT0UpumUEa94QMJ&Qcg_l6aG3LcSB<%Ym`+0i1C|Dfo{xCQy z5E5W6N)m%n42kkMZ!%&=mK13Ul`sq=QP$*N;*y{Rr0QO{_D8ZNc3~4$N=%p_l7vYh z8Odm*Xro9GB*RfG&eJptG@5vx<57w6*%=oEZ|v7b^*H!=VAU^Md{?(!RNmc}bdQ6? z-UV9V%<3~_W^te~^>H?bGukdXI$O;cZXgY)*@Q|wBk*u6;FzEwRs>IC7|se5Wn?7W zB#JCa(2@`Ul->o(b?#s6ITd%INJ)O`jP9#*zxt}9?nH=afIviaBh3hsU_wchc1FldU&$J(x>lgMcsq&jm>l zXN!;J6M5F*w2Eeq2YZ*G?Fj9g@ zad4td6qjr<7(0e^$%r|songx3b{Cdy>enNW{l;|N#UOEI<78tkdw?+zJDSy>AmuhI zZU&1@fL-GyT4Z<>Cvkz~Ne0AQwhII&+clOWC{*UNWMX|+%k~~tyYM|_Y08*Zvq$z6 zue7ox&aBYw`bV}*u<`1)!f$$8oEQska5AH;kGB8;jDawdGzVrGY%<9UAlVoy(ttHf zpcqY{0twa)UP)A1AOcYU(M4197Jp2bO1+pH)n!u7de3|Qs@ofWnt)7|D3RlF@F_T! z=1>z&@B+?bf?zaa0%2l!j3ZHzXBkvTohq4iUSt&-*s|681H;j#*=F1-6WM&O?oE)Q z0b5aI&r0|bzlfw*MM*uBt}D@PVNYPQaCjW4EY z8rWhp9MxwoIj1}PWogB7UgKW_xn>~hgn$@@TNpI;B*%+%#4rVZ#F-f3E6O9@P zLJ)8q1=pM>S&E^+SqC43z`^3tB+7jVj!Y52aZ$9tO^>6a^Z%6T(1VqYzvvvdIH>*b zPmyBZylLSqv4Wcj2qkdRWCCxGVnEHZID=zw^l{#Z8d;ObSjM! zIRT}>8zpgy0?$I=Kv_{Vi4mA6Pz?AW0z;DEse1393|NKZvkNin2d;=|_#vq91mZE<#p=L`}Fo+e?AkGBYV2)J3N&2FZYkPZ`AMq(L`F`+a^f&gJAni5Sw z1SfGI{){AI0!=YKYxT-nm$)-Wotg@3T27t%a@6I|PE6?@6zk=vIn8d8%0Sgik{A$c zmchW)0GEyii@+Eu5+VfAh@qsBWCYAa(Hy7hRNv^C05jZ5J{G%Mbu;@3Gn&|&IIlUPB7pR8gVe*C`}tt0rhyeu2Yc&#hZ%zALlz# zA!;7;{q}daHoa;3UiVwLsPIb52!cq1pinqYLk5i}NmMXOgvc;FL>w4qB0*_^)^a<3 zF8q_HV=~QN*z;Z6Z?l&)p05AJy0}cbgF#>}o91q8*RCMqbuHl2S!y;h(+1Rln|YiA zH^)R$fF}!{16W@YCuoR1A*Y9uypaOQrzpX*POq$US%JUq;Tv1+?AdBYgQDFd-}O2k zKG~4XuTP2y95WOmVU#n<`7MYJcu40FByR$T!bI?*k;Xt`cwgyGg5oV3nGyz8|I0D7 z-RubLFLB4G zxzCPoyYjL5sQV1DBP6`(h9DN@0~B#r!7x|Bx4 zC%S?7*ofjLlotg?!XP3RP?SL7R~8~T@HQaQq(DpgZcxgorfCj+fjm1J%+Eh#-+@_= ze|#H0Zv(twy{u(C5rKe-93vWG12BT+2@_#rMJTxmker7E9szzH3g!^=4L6gZc<(`p zb6N7N$k4sg*oSu_zg>KJdH9ZeoZO59>9JVeBUPkG31GGbi31M_DpNS5BOwf6D24$0 zO@kTd*F^-s zDbOhbXb~E!W2lG<5T}3#0b0{Ua2!QID$aMYxBt9p?)AId$F;vRc;UHcbI&d-th*B$ zbd~(G`>~3V@h)uhkdc*0$^;2Rl!78E!3xm)fkE>G3NaRxau{0TKv$*&;C=Lt=N>nr z-Gl?ld)Eb0kgk^zBD03a?o@6@@}j#r;{+L3ebo($aGLz#mvYbVzs?q{YT1f)o% z0|` z`?XTnQpE@~CFFP@izDX>B^s))D8};=oB+tKaT0i*0wq8u!Y>SOIifOkymU--YT}mn zx-%!!{Drzpp*hs<2dlfFav$syWC1bIMi?cRzXTF$VbJ~r)jSgk2@0A6zZqpc0_fgD zIr*bzAj)pJxAEzny=!+o)J(EIxcQClbZ8M~ZE-5H5Cwv=6d>gAtwNr8G-_(&!Tdx0aU`Ee2SVpo`Dwx zS7}RWU_7-jc5Q|Wwm}O!qxMg-@AzC7Y>y@AUI6Iu@CjIIAsM$z-}BZyw?X@EL*Y7=wf60Jp$M3LKn9OyVIKD?;>2O1%HR zLlu;_B}%ag!KBgDb7)8sMUYOVkHPCLz1(p^fHFhPM$mWQz<^H!l`0Af(a>$e7{O78 zmL-hFOc?Zt3MN9Xp!gN+kf5Bcu@KPYl2fe^t%u!M9a{r1g%PLs{|{( zRkb}T@9=CHzV5NG_24Q6ZZ0(FZU@Ogwg1bNL@LVG6UAm!{f)L)G>#X=9Q2-*+4TyV>i1JJ7gTs;0P>bvaDFLssdwPX49yiJbO z(7g;2^$r$JzFy>nb{tpVGF0t7@9C4?Gl3o;47DwA6UmG*G(phNJ^>j$RKRHhgKRuR z$vos%IS7so9pknRP;gUTic98~DF1I^*PT(;k^ z+=chH-Z`(n#HgI3b$KVvs4bS!{Su^fyD;k*Z}mnK1siUrq215Kn;3AQ1tV{Q@~+&K z5BW~$Y-d0>Lv@K1p);`7rT%gyvxd+*z?N(>X0<`WuN*vgL0Gn$Zg7)-Zuda z4ZD!wayI*9kxc)a& zl}e6-d4fv&fTIfa6=yso{nVfCaK#7>cwg6QSSzwwEi))O0@~7{V~K-eDV#GD3?QU| z2%g8Gs?O0S5?ZKfxR2t3rN979?FGZd^PhJd8h@zm+KbCSH+EimWO2oyGXF}7#K2oE za6`{wfGS%D_{HM7_-R)DsF&aD>Y@ZT+<@shxIu}VNjdx?jS}>uOLG1nQvDpu;b43) za8IG138%qiipCUB^j=U@O74HMa$?;UCA&A_U$^O#_l54Ke>zsfYUNsb1)E+4(4 zI~go`zV~w7Lh0`+2xM5f8&9e&$ z>o#;>8fr&E$+^2&EJD&6G64{5aFBNgXP5>ygQ4bF%E*H004Ee01w~IbOi8(WITD+O zpR4rB+^zrbujC$V5WZsK~<=JK6*Z8RG}Q$YG$INmb&&tH*K|J~Q2L>gAcHTbF#a zp-oVH2ISQ%jL<5|%e|uNPnXsu;D!JVcP5Nr7f2}5n*>U}V*~PQ|r4 zp&JlX=UA~V?=_a{7+Opk#%FsLd@#+bg3-`0_=oD0I z&zs0p&u81NQy)F#vj?+GTze+js^M7X^OZ)vErcth;@ zPh!g!uWdZ}q}y3kcQ{BEcXZv4Pc(aeks~I|EY}L*niT^%M2MQnlMQFH#NhAhvYa_)gGl;*Rtn zgGdt4atAdo=0RYDXs=cWMzyUaoH0xF>xG;vEjbZ4h!_L(UwB zbO1)Pl6*!P6lcKU7iq}Gd;p44ZdAqSUghXDIp$~jYrM@ewRe?nHFdXw6>`qzP+~tN zuhmh01vxucmWzM{iYLT|_GJmWr-_1>4t*}r0uRZ32D(C^CI?qG1sv`d z!L3IV+|ly`2tubMKrY-bbog>jr6z}eh}rqFUi0_Imgp`A!Q5oHcVJ^-9FE)@p7b;` zT;3uC@P(oO816$+&`HJdQ2B(TXu{zJ0?P>!Xl#L3(NhSg zWc7EtgF*BgCBs;0(vnMUdPpN#5=4D`tVNEUYlvtC=zM1}N+lx@U_Z2t$iOUnvb|i-YODL2Pq(U2D5zUS$!LQK86uxjjKBs# zfk=R;mWM-$LAM1bv5@z|jnD*7LY0&^LG=aVK$kpvp!mW1@VzqJ=(R^GRhwGviaw#+ zoS;%PezBG>TKn>!1{WH#1i*e8iUK&ahd=`kT;>AjO5{)o4MhQN8$b`9H`>a5ZEtCG z$^K`rn=kKw`eLc=x$X}fo1t1rxb4|+U$hBQPUP~?v;cK-6e?P9%?x}L=ysGH0-yl` z8wz}a6k?MS>vMTNsc_@n*VUuPPtA68x#n50IS+r@#usTy?j3?eJ_n42v>h)(&jJn2 zT%<@qUQfQ04;`}*kEuv|R8~ryKO6XoUaVW{k47JDI69=sx{N_39g|527caqB9AAuK z?^!5dNpeOV?gv30l!Z&+P*TB6P%nbp44qBOQ;7UN#9427U zP6o{?U^F4+1%ZM@K>HB9i$d)Oav!J!9cM`iXl1y8SN6GG^F|GrHj6h~^~Q%k*4?y8 z_j?eSa+YoewrM=nCpfr<>hsF0zy^5)aQe+qpXcO6In>V};UV`vL3{%#e+U^MO#^u( zDCsjGD<6ad=>$1=P#A45-TTV2SDT`%Y|mz9u68Y~`y&YSVIvLEHd{2zvlHdfNf7hK zie}~4_R8<=lM07&6&y16s96HhfO-ndz<}#JP;P^u7vtqRAqn0rRGA^iNJBcE{}3F= zn`X)FcxO@i~ zN?SYuK8bt{3CtxSN&?Iv!sW~9P{)M|o*Y&ChomGTRqXRcPy=I1Rn1yvZ^`qQ58fCV zbhph1u?Ig<&P2%pwr$T z1xe84lnMB7iYT?LQF!`*wX=TOP=8eBc_nt<3o5XeAE)F4h32%<^F2N;!lGv;d~l zp3HZ?(Cl*G{8fnvD%UO}o#u8q-kuoev?*s9Gs6uf3VPDvKXCh>!x)H~jYbqIzPMap zf|@TQ(lD^ZbKxPWT}pwXoMbh6>v}lTPlbmxUNWKK+p|MA?YbJ2MUp8|V`o)6CuwOI zQb<63AD0gRIDT^fEnEpBFlZAtQ8>6L&~k>#O|1~Wfpl>q_y)eKA##&fyTqmO3FrV#SbhvzwnQmf$MPS!Rv3cjmaRM?{x2{T~)6IbrsSb`OA|RluqL0 zZzxeej1iLOEI?fh!aD+-MYyIXcP&EiGY*mjsUo;+1$v2qL>v@8{J{YC0ARRw;mZ9% zQBNvRkBT48|L)h3Z|4Qo!tK@(NyN*1N(W3NJ88%G;b;z(@S!mYi!L>D_f3U z5#Q%F{-Z3BeEUevfxCA)=`FQVCbvgH9{`WDa%oz=K#Rd0Pf_5Ztw?~Z0L?)e4MGpN z>Be~E#TN<)yQe|w{O(hYO4qoNbN+)NdtMHRc&z)^P4`3iiBo9E~wbI zPxV7)H@EeA}ZQ90O>Q#P0k<|r9{0ry#Aw}OGIQ!u0xd9W#eABYUuLX1b$EW!rD;m|!kgvpf zqDOSwr+ojbgM+!mCVN0(pXG;qzwV@_+n%gj*KUqDeIen~n*Zuae#pgqyI-3N{51F5 zhM#`a&k-^IKR?G0X*N=$uUKXKh=Xkhp1$3!Y^nbrruZSxMla|)xloH1t?$>mnq%YE z?f>N&en|c&zYo8dIkH~Gj#J*nZRu?b#(D*aDV3V9>S)WYprM@Lha^-V+q!wDHv2|z z9lYjZ7qV|KdR5NE{Y&QeA!Qq)_x~W?J|Z$lobAFu>gw=ABfPdpoJUOLQk`;cwTPM`kHIWB@*r9H!69J?_Xw);yU%7cR3jif$<nS0ydL(@*=MQ+r2IoSvk#e~UDYS|%r+5&i{@|pu*mW2 z!Q6cMJ3FJs%b}UrheXbPS;e{XQkfQ!aXWRIHY>um6TVaSR3ac98oGIX$kyJ~?iYAN zeE(an*Z11gC}a=D6r^bWHvjaSVJ{$msRkTciz9lS8^?4I}{-M*>cV$8t*?0i0C#Ek1qrmsu9 z&CYJf`@_ms*TQC8echeVOy@(+^By_F=56}KE+Lu7hdeDkHCLq}6C1uC)-o2KcdJ|2{0i`t!!(Z%DRp?yo$@cs zw0_$-*Yvu@n+st(=w4&=)UX9QPeU|~4{0zh&u@1lj_59zkUuM@^H|T^O^R3d+W=tG0k-2jx_l2>_hHQF3wE}Us|@INzy z5BZ3mI;7&MC*PI(VS=t~r*+@`Hz)8RH}iE~)V%AQ=@F&AJpbs~eI(c-baK7nZvAtI zSMJyH@I1AfFf{Y`5NXxJpXXle-!kj2>D;W!QCz5kEZL)RW3PBO;hDaN9DcBL%Vz1y zkRBVaR_#A{Y55R2_iohPbKAl*dk<-KuJeVhJ-?7P*L*N##V-rHhspzR%Zr-~_iNWx zWVt0AzP>FqllKteqctBl_&LkdY=ydN3zaCd`rkcw59#*f$DeGNS?*T-7b7>}TP_v+ zcTU|yMn7J7aI!Xj=MCqp*N5#@>jZNy0~9xXhvm7;o1vSzhcr7={pXVFx6hq+vDBzX z7`i5ybD4&MpjWf5OavC1iF-&F^VHtCh;b{MpM6@S;MSGp{%!O2kY?3y-ktJ#OXX(Q zmR>l}^Wd++rt`p_KtI33@1#8C_CUy{?IDw9c0`UpD^PIZ^9$zo(~sW|d+p49b159N z_K=Gs?8UbJaQj)A4(0o{A5vpeF!Jr`;`NM;OlryXyro|MmOHCMHfaxOwS8Cq@%2_L zsyWOy{CeJn#(%|}Jw%MFnsLR~&z|Pe%z3-KQ}tFMR?RNo3Ffbw|Iw5^q-pk+1G)_# zx2ehY#U=WToRjgtGGh-JH0pBV$!l>7t8Tu;X6W~}`hRu89`dx+v?k-fyl^zj(@VR4 zc)7lIF#Q64N{nuADP;5YklP6>9(;T92gC5D%_|gmRefhLd4&(%Y_zg4a43}1^^nm& z^vE=R!@}EVKbwE=eBl#xu&HMCQjw~E!#rCLsZi(Cr5;$HH6tDl>2a>dl->Wn$$CiR z(JN|{)&J0_=go`tj!asUD;Q(pM_&IobM=rxee6RjuG;iwLKUX zoMWW#7v12OAzvJyb8FYXah@JBXJwwjhgvPjmW3#lf6dr;`9kbcglL)`a$&}}Ih9AB z-8fI0wZHAQ{KkLpEInkwty&8-&wC9y_~o8m**h%D@n4vvha8-gBWmJZ(X7jj4&9xl z-p3(!7?g~8DCX!PhjX2r@_UC$6|U6(;#tL#!n*&?6g}i6%!7P=AtI_9YuwEb&i6DJ z9)p<(`wTtgbdNv#Ryo-6m;R@Z9+T~L+TAyjEk3AejGq3FIc!uVAev$7N z|D)-7NTo6Te);M1kKgsBhkqVDwAR32%MPlC`{D8E5Nn2Sb{=xROY^P6ZWJi-hPjex zsxV_tuto1w$xswqNGInZ*8S~9ml#>Qb;*Yo%=)(7%V2zAcjCnB`fAFz!Z9}w8JbX_ z(T4LJKHbY{qhyB|GQ@9Asv@|UVM3lb;WRd zkyF=7J^wP8Iwoj2e>1F7jyi+M4PSvqabs29t2{i*q<9(!6jcEAK%DS>FKv4Mm($BF322)KMmt69HE+$hs5bSf3fr7-P?_;{$HGohjhu)s^@~pHWdfhkt}cT zUd|hASn$Ega^ahchx~$9+nDfekJwe2Ys@%V^79qJcEWrR_oO*?N?kh)Q}K|NV>TQh zO5MZv4Jo}Y(}`!-g9+B<$k#o?&OHLp{n|a#PMKE~l9_nOtv_mPSv@E|@0VxFy~fFv z<%7wT`>Ra28;;y1HTP?McqZZ@#%@KwFHj@uOxr^3s=Y$-5y99WnU4R-JUry*H)BVf z&0nX-CtXHo8G4I27|hKeu$KS#)9{dAw?wtyo3VE1hL2jc+q)|J{a`K|x;Ve9) z+4x!O235p9y)vlj5#m71^k7=3WqI*cH({8Bhb*mnsma;0@tK6H<@f6+F8@6=iqY@n zIz_L*0KZVr!9&QbC${Z6P-Xec8-+Gvg&uzWPjWsaI82Gvq7)MJPVca5owH^$RW=!xU zUhr+!>Nn+js_CR7c{ARwoNuyO_pie-#b7_A?J-C5DW4o`UF=G} z=pMIkUYX`*yF4)gY$Y34Q|^g~1CDFPx&P)GO{u&$5+=lPRX{7#j^EBzczaAevjixvDAu23$^HBaj68mMur)&U>t9wf&FJdptwC8emjWVN^*5 zqlyBfiZ6`cLW`yAd+6)K*EBk8u{D)fDylorU-~)orI8o*^qN;O-=q?wv>O7Vsw}ON z5>Qz1Z5$`YM#fou@QqrqXK9< zD39%3Sn_cNOmDcJX@z!U8W`*cQdlr9#ryfIHymsjxv?LoLOuPd@vXNMHHH@RQ!UgIGJ zYH#dwcGiqK*!SA)X~f+xyoce!z7eQrHd;)9e&wNkZ)Pjf?b@xUzvRoj;=m``9ce`0 zKftiz-^}%$f+_It^4IHfmP`Yx?~YqEnJC<7d7AO}4=`+Uub8sra$mLCxmgp6y+0pc zdv3YH13o{QX1VtREwt7~fKl!8;c5ah^R?)Rv&A~xhP=Yr3YBN?8h^K|*nC5=r~5kv zwl)ArIi)E8g;o=~@lA&7)0InR?S6=?>au0RH%O*l^#W@`H?BTFLTh6bJRynU6lLyy zEq0EFasR3ooWM^lJNX=Y`^R{O1;+I6X+>Icl-kmITfTM8W%flMiTa{_VpVV`4(#`XC}w|}DDoEpL(f^{og zSSMrg7pI5p+TVMSrD(ssJ9Z4%|602~jTrmD2|c!|J@t7?1X6s)m56+^M)dq>>gi|k z+iUbq9b12JLT@WvC)vlYk;3%{AGN8BUszjydXA&5i=}QWQUJLVQW+4t@}iPWF+;#t z%l$$%7BlKpOZ@!elT1JEZdZP9^IVya+YsSrGn!4iGIjL4A%ylkxwxmoZy+YBNY4GP z+whn})sAj0sJ}NfA3G=Td2;bE8IW*rQq?ail9g;{r3QZ>hr6A;c(5LSB<(h9XYUX;P^KQ zU={xW{|Kq>N_w2RiOu1Rwu=r|*yG+1bq69l4>A}pJ)K`Xab=O?t*;H$?$riTXh~qJ z1OWRY+@$#DU<9)LM!jawI(4oR+wXLp?0buTt=<23sqqgv>;!pSuVmYEX;F)@CCVp! z*YA&WC+}_Dw(0K^=YMGimcwgh1gBbn&-xoGNV8eFq{c|n? zDRt!gIX&-O$UkXfDQ%hR`WI9ERRm%#SL^JTg^JGXI4AeZfjcK8rdi|q zf%Y_{$K2hVA-k`U@y-rntWz~}wJlbs$1W%CY(Hn_`G?!L*QA{pANB2}-4pPLyP?XC z><6K~3J)Ps^7jP5w?+Ft7wUTn7Wlq5RRQ<>bnTI6#Yz=AOOE*_@sICMYY(N3o-_b| z86L7A&UE^cR11mcKg?SZrrl`81a9u~K7nAMZn z#BPb@Epe<>-J7}33QgxnSgPdXiWj{#f9LJMhs6!8ieG4tLcAHuHwc=2_o12ky@IZ? zZ^oxN_KaUQePqizd9MqDPR!Kq3}o5C^Oe7ou*!!ahBk4bK2PxreeoWC`vdu+TCIHl>q zHwC27rZC(Xs}7AX%FiDTD&1L&{|p1WgVky}4}dNAh#%iEf1 zlZlj&?uS%s$g^kaSZy3@_2jhbvhtTqyV1_^=-KRLO`0^ZeS7M{)ofE5YPSVa833+I z0C1s|4H@^ZY|a+37WqQVmsU2mld7dl|JeYU9@+C=yQecNEzJ3Bdtlj+k@g2C^w_F} zk9vZS|FV8ZPO9qgtrv5xqQ^TI&9|kFtv@)SwSzKmy&0Qanj7Znel@Q3qTWoMTR%_R z^Llb%J1BGb!_Y$8@Z^PgGfOvjo%CJSoBUT+r0FG%Blbj@qD=xfJbB^kff3q5UNS6` zv-i{X^$Il_cd2aoF)ic!R5EuBT*ym?t}ig{s=wJzhHOG}js8A1a!uWFr5vM;Q*$EC z+dQe?@z?B|_E)}R`>esv_zV*SaQG~-O2#S5Vu#)|$meLbr2Zf--eMKZyxa~aI$X_b zavPhbKPVN=V732DGdfuVnr5KVfuNAQ3D@Rp)e6@sT5U=!xX~+9EhpCWGT^+Uj;j2R#-!v|j zeNhYR(~}CMzw&9v`}St3A0MJOr9w4?7H&#jqv$qz8y~L=c#cQiq<+hFX87tR~Biv9T@)0E19qrt+k_D zdHRdgk1p@?>DB^eW_{g-?Otp_u?`9E-pC&vk^^{9>vWISHLFNb`1YQccQ&~^Lo*C` zptkF1C&>PhR||&l~Pp;nB&rGPsMlc2bHS8Fj_3T;npLWe4vY z9@+Vk?d|)9r|&O5+N@xXYIllu|LN#A+nT?gt9<%te(mI%DH;USB~98k zZkx0Ew>v+dI`#0{&wdy>PFZp&pLUvSi6;Z)YEV#DwVTxSYOP^k40yGx+Ns6W3{?** z4Cw=Gcde4y;;yu-OQI`&647(#Zv)q^7!cK3w=Sl)vSf2U?M$B{b!p|*mo_2M&CKYR znP2q$;=J>+!$9e4&*(QrVP@al+VMUM-5>vY$E(v9@F&cv47G0;%RkpKSNZtXoZ1P= zD$FWgw(gJH9d4oh>T=@Z`41+%oH|*->u7fERF!mp+tT6Vmh8IQ!ceXba&e{p$K~gg zMNe~UCwlnhyEAQiVW*^h)2X7vp{r5hPnk_q$&{rY)TwV~k%B$NPFEMAu zi7(!8qnIa^Zfk~m%4ynhK1D$F$v^b^;r8u5Gb^-zIV01(HdzNQS743LsvYnBX@{HN z?Y`YozS@JWvj?=^m)XQF>b+6<_{yB{=y??>^4K9qhDj@9tZ#p+v_&&eDeh>8$+GKS z{{=pw#-+~u>CN{`XPH&7ue44P!1Z~xBUM(qKltem@r3@(l!}dy-fv&Dao+}46^*6N9WwXXpTQ$u-{-<_3Rv)go_R^N#BS8f%MYJ>guXnGzHZvnSw&|rmu}j&- zxB@fTUCO$nw05?8o#(8060<6mna3V^aO&Y??5OFwxAuC(^uXoOGgw?hnk|Fzsy=*x4`On7`s6$QK1*zop)b*Dx zeQusQzu3APzy7sm-rb$i^Iyvr>SkH(ynyL5b@kLIO>=1Sjw;v;>v{3<_gz0fp{!nA zNIOa0;iM#(y5h&~cd>gF_P1JLup*gD&02Cx!FO>k;KEYwQpagqK6_T7&FQjNTV33s z84Qe<(@5H-e)CAB>p2|pqL54_ygA>RxI4^b{NhWrA^Q7E%UI)p6EUS-uiTHgu)ELr_h%LtTeq-L&YI`5ZM?3mT9r#X*WaopY@*f13hu>qhw(jY zmnfY5r0%O`n}&Th=r?8YEg;Kxabr%o@8MoXtm`psEdN8V!3`5L{L!S#ntRH!6&bbT zeU`aDt|OklnEBhh$39a&zOtxxf_m7KF`{I7!`;sso+=!_pv;WnjYr(w zr6{L9CA4GZLzC>ob8FYh)T47=X-Jv;<2O6M`u4B_{OU*A@xG}FmC2J{ovnH8a^LLs zRrB=RC``6hPRZ#iq1?a{yEI zrZq7x&wwn%YGbW2BQnf%Bh^7

&cQmm~iG7Z*gY@YLV)C&$^v1dA=+QPUO^XR*3M zVW{qC&!qXu&X_p)>K3dP>&w@;+z{c*5Ii9DkV0e)@YT{9G{Ko?)hGRed~3OQ>)kaEBD1BZHl)vo8>#C(RLORfdR9~lL741KO^(6Yjsa>l#ciDJs+1;q! zQf%facDPa3EKcN0V+J0|XwAe}=q0fvh}!X9f02_D^7Y1uK2yElwb+}8_81EcCx-rc z%}nn<#k=}vL`O5Yt>+Y@6SFMxVgMUwF-WjUhNu*u8(iLy{ORRl+UWvhz?nkzFz@^9-*5K$2OgUV zooD$f4QQZU?u|l~RM!8kQo@H&#M`%L1wlsB?WwkhbbNb<>%xu0g=buj(^n+?(=5f` zbUA3KMDEqD@x?O@K(+<7y>L+#D)3tEAlENDg973d;i4*>GLY<{Nx!cT1{N~@$y1x~ zv~q)~MGBX%_jJ1%xo~se_!GaIs$L)Tv|y2}&KcTKzNt#h#?+tVvQbGpBxfj7oDaw% zB|AdrC&~KwaCQwG0L@U3xGy_3+lnTSwL{EPAZ_ruei~1TZnSSk~G|bh5HJZfJA$ zT7v11l`PTmcF~Tor@2Lo&8wb$T)S6?QF%6X2?^T8N62M+l{|YM%}y^?D8|@kv+^B4 zIt|cgA=&{&LzF17@m6PZ_g}pDV^3k5CwhZ7#Hi%u(X2}SIWk<&Ly+W#dZAyy9#5jB zXqSy}0d0zf?SFWn$>^|as-=#BN~w%UL)Al3;U@Mt{>}VMd+a)mww@-lI$x)2oJON5 zG{-+;vP{*B1!vZgN3%ZlNBa=3tV==2s?;82u*lc-V;~)&n7fG431X~Zv+Et;-B=_G zzdBU_ZolmRj+={9#nDAn#4t8Cv6Sog;qn?4@crANb^^^WKtYxjQo7HRlRePKm z1{uqED=S|E&!gGowZtR!@?lWrp^@javY+Us6Ozm5v?Qo-d?eAXtQ3{G7=zW|B9CT+ z*W+XieYKBzG#k6dYo5TPh}Y3k^B27`FRHYhI|iV31sGscaJ7z?#^@K;qwb)x6}u1?waw#CXW zs%E_VZHcv58^KqVUD8M3Rbv;!@}#0AYbSj)0B>h`KSzaZ5W7NIO&P19 zneOlJATjb5WcP3a+-|$;I9ekm*pAvj321h$XSFQ1E)i)28n>og!qiN4gHHORD%zUS z?myW@s|cZrX1ps_l0^niJZG_jI7jruiSbU>9xXy4F4_AThbm#eJRxMSHZ4sbAE>SV zt?3}jBO{O>|D>z@`Rc8)Ri;dl9jYY81imejd_0H(acRQc52Uno z;Qj-?`>Sq8AT54=-+Fu;p7qnA=;D5YFg$Scn96v6pu&v3zgl|)l5@_INiR3m|G56h zh^D1KE*+aD?EQgC%e0Gs!24vC{|LmEdCXSLuEmSSbQy5GA-(Z_;Je5w+mk^_%RPo5 zaaJ1*lzvWdmDQK7EH(7ZKQDi5NtNC4zkWXTQO(2Eeh%DN2FR=HlaCouPVArI(>87@ z{C7V@AQClhR@AJyZEBUTQI0vMeH6I&--{0a0MpWV{tx#kD;O5dK0{lS>{|?CK%%{&P0{Q7m!}_CZ52*b8)p{kG4%xjctviqcRJiP# zyaPU+Tgh9#B9O8eEkr@OObp%N3nqv3A*iUwsjQGTS{{Yt-3b^@%vI+kIp$e%51Kmo(D2)nFJu^P*jKGw>X`cl z6iy54Qf=ORia@SW9p2BJ|1Lwd?hSGc9(-(0>J~O7l(bx14(5b<8Z>=or?{01HNt$L z*@lfHbG^9IX6Tfw_F!hGxL}3rr25BS?R$$rPDd;lFl2g$Z0~v(koR*M6@sW@aQ&ES~%iIsz&Bril^d^PVB!H2$v6t?V-c zN1myG3l|l3PkVOqc`~ZfIQLPLVQ+;qUrhOZe&4O@7U~02kpwRJKp6#GxH1|^kM%QK zMkS=wgQgcI)D!9~DmQD>`m)=O1kPwAL3I0AUwb&^lFN5t^01Pg{GmBBDnWV8w|?DR zNt@-qS@0Q!GOoX11-?b44x>fRgN4(&?}mu6Iq+`2)k z_Vz%?`20e{P&+W91LrUtv=4?rSX{9LuBX_wr}w6f3$g z*ZM+FUjJgT=IoNnY$rF^56Quc}z)!nS6;>&}0D^hDspc8>z?VAwV(z!)EYm!%?*R-Yc|bV9#; zMf~JdD{@#@)X^SGWvc>=_3`(F*J0Bm0Z4Ug#4Fv(4$So1p%3mhk3i->e9|#vg)tRA>-~M?sT21b2F^OV(f0={+-E>` zN0K3bn_HTz>`xu@By?}}{?60yyQS?6qyQ9dx$$$05#qTeLu=+Nta&$N@3_kYiY?xs zHo5TwDlPN(g2_R`&B+gqaZ*)~ZLjmML*us|E>`QyiFw#_n*x6`4#4}+947@{xa3Es z!|z4T2;_^(3(E+D8f<>ve)T81d~+KHmLHh_zktGtx!)V5Pz^q|Bge<@&-DItaHTsP zI}ZFUbT<$2qr+D^nvFO|Q#@+IjFgdOah4P@0h2iR zg{1|QmKcJ=Xq1ss1E*KO!C*Ig1majaw-p*+&wT8=y8CL=3DtD3b^jB(RFY@`H?k;) z;}Q*|(L9gxJjSpLgTenXf<)nO9ExFqND4&Fo^bC{f@>xrwC$@D;XKOV48e*L&6$8O zjuBACgmXNL@kWxOPy)8eNRxgkN`e+}k3#j;&9c1t#iOIZ@hUwOW;C$`DPbf<@bDj2 zzyy>d<*y&bSqfuK5^s`>obiKj0#Xw8{gH2%>EY_9#i7!};HW@IfVn713`Q{|%HzDr zh#BEH(G)6S7)GM3$-l%UK?_J#u;0d>s&0_1iCx%4l@b#sh$LYWNJcUmDcUGf1j%p| zi}N(i0*xk~=Xg|Ne0IhK!5jOvQ9TYm9$59u7T?uv7nOJSCEep7HGm5=+z(ctAu~&! z%&do`ureFJv(=2@2GW30O{) zTI|K1Q*jrHl;o$*=)OAltFJohPK1aCxPvdck!A!*Frg$$a&WLINg!#Q<|)uwoM-|C zEU`GsqoR>;BQ3A>!Lm)8`Y#5X9-geQ^y#}=QMY1r?}EfKRXK6n5O>DKeU^EU4;rUT zBF<1G&hqjZW{i@^VR04 z4ZdxdLS;TnCgqEUV_~hg!XzE#EoGRD>n(jE8)KEp!3;(-CJ7X2qERD32m+3y@Wt&s$x;jj zU*!%y27!acqe+ze5FD8zfa9WQf14ghN9X@3)1e0|8-LL`ZgEii;h!SKzIoFEb8!SW z5%5LxIB7D0w?{FcW?7uUF*y1-Z$yo(N#t=*4wT4JK19d>e6TND?9Qfwu>;FXi#HWr zSG|6e?s^c1BMsOqq^RttMF=DqP6F=^mspdDktmYGL5L^<7brL|Ea+pAqdk}_BI(WZ zGKtZN_lj=LwsY<6yQjK7uc&()GOT4Oawf)!o`<{N(06p$2@G|!AFP~!Qs9k}I0a#! zC~%;xD4N6wOcW>vd=P;lN$^y?_fH0_!tvRKnDql!#5DYoH}^SJq@T0$A>G*^>4~;D zJH&Gai3CrRFvr7KcD*tI#%4EDN|cHTEF-ZD$Cyx>BSC;L6HSRGAcB)P5PwF3u$875 zpS5~rtxMdQqfSkQH7%!3eL3p#XD6m~*BuW6RXNlKFG3#rX0^p|woYdC2L#fLfuziU zi6oj#k^n~&6+zUEMo|uGBoGXYlfcsuAOK~I>N;>Flyxo?OT25!Ps-BCGHdbPkG2o2 z@ldz#AK5WUzOWA8BHzjI;fPGtH(u!gn&IIr;sRJ`#>fef7{Dc*HIa+}nuW%3n#GNv zd{ou1_en0eMMee}(4h-^sOd!zFN&0CgkvgkKmyKT9L>;hWFavC@h2~lBFjQXz~q;r zR4EDO0ROFojmlHi*OnMijx2+L!wP``4Soe)iLfgsi6wy)N~C#~6?qeCVmXW_c_0PofB|oQcO}qnyb2fob~vH3Z1YwN zOg3@v1}@=iE@6!7M zP7Q%HvI!6nr57pEL8OREQF@VHr6Yn$N9iDffPyH3h~Mwd>}+2E?Jh~srm;e>P7WXw zIFGtTYq#uC?&Yf`uly;NeCp|CwyeI#QeY+nzTIFbmRD((l@*SI`7I&j!D+Z++=8G{ zuxe?Z6%c_Aj*t1K5O6?>4gY2Eu9i={*YJmipV22LuXNlB)h?vQOno!Z5Dw=xN@U#( z96Cl8;aRvC0nQ%lf?ZDWJWHvR-+47vRVtOhJ-GIK;r$ird?>uJ>A|J7zqk0)@m;89 z5fE`zRgH(hbGX3~99P_KN@aCM69nA#h?0qJ+$6Y4ET7KgpNtRWJm>rK-h9xhbDmP{ zkw!IECFFGMvB5d9y8B9f`rxQGh=#isUAwu5kE9}99>pcYgAv^v_>|xj;Ffl~7#@Ko zgzG7{!g4sl9H;unMHu5u>*dC1{0v4XgHt_wB*s4;OqjN0GB$$2l7-oWc3D zuAW#tP$=R3v?ycAMURf?7N;Th!mv6ETSBH}okqBk)p(ZST?k7ep2y1maIb&7vGk(3 z^e@MXqvH?G>Xf_E#@y{bEn3TQBWsw>z;N|7;jomvP6@P|b-}x*B^NwLw=60G$EqBq zxp|#)>26}irpaCTXap{MOsC^)kMkeiAK2jB(48rpUvz{lgp}6N&cJ0xo)H%Sf=P->GYH%ykOx)+<`JOiPYk*` z>>_ojK&j-j@dIkkAMN-pR1%fpRv;&(a1=B>!gsWQ#E#^KF~maY!&ZfEa_RU>C3p%! zgaUVAVzr+m+rD#o;>4XD7(kjT+>M2wNMQB{e8kqXmP6jX_ELB`yS zix!c!N~4uB@DaW)^;5AK-&F4VME|9gv)!h8g^quMgo@EDL4x=QG?JPOPBeq47aUVY zg(C;|AL%v`5qh0MemdBK^N|Q#{}T6$Mdhyl(F+yMMsK?{_wr=N#Zbu-9;fvk0-8g3 z$jzyg#EC4DX~eGPG-8nQG>jhP6cB|`87=4#V0a+k^9py%R;u+tX8M50EpvC@TNgT$ z6OSp!c_rf+O&|Z7j#UzB%DAkN~{oMxqKC zO_!z%u>KL`=5ajSqM&JrJtqXu>u6(Lgoi?s~*l|s-o zK`JYNOyE3{_m1DtW3;w2w(0Q5Q8U}c*v^VRDiGu2T^Cv8(G^(|WrZkSR#a3l>=FX-WHQux5s%QCJvH66_f2y*>PR2P2$yP56Vl%4~lmjt7kQ^-Q zQNWodGf0;M!^6NUKn@B?P@YFN2Wfv**ARuFS;`-OH~7Joo)>)!f8aukW1tr9P&02X}7AH#T_AK(|=?X+{csE)BALZd&7@Q&eaX9zX<2 zqg42fuppo{MMjo623e2b!GZqF+7HfO*)*~1_ao*Vzx~0ng(V!{XAQeXs?Yjb26L$ZieE3S79U-83P(PH3~j2vgQJ>%g~kS2?QR!g9RpzDYa%y$q#dtE?#)# z_xl~cWD)Ov)Q8V*w?31|8yp!dK0y;ALMBp5fc67P(&oalD2#>|T zAxJ&66a<@+EoBa6Es7j+kpSihJi-zx{6_>PaBd}@cJautYAz(kGo%qHPV$jnbesT0J7pb~r^=~_3}r|oP>#5q-A^6RlEM-ezxDl`oFC0|oXVO*ZGEwC z7ksOa`w3B58rq2F2+}}x19kwUDS#8A8v!F;hTTj_egU-hkObp=5utMHudccE{q~ic zZ-h;axqh*O<4D#rEX61In1vSzWvL=Bh8TkzL2+QA7*+xJj78#-qeUcO5OMIEEf^Un zAC2|z+kCq9Ud~m*?S57MT=A5x6PgJs#dv_ln--U%v!cu^ZUzer(1r{H075NLi1Ps7 z3$U^(%5KetK&ORE2__SSOlf+oa|h|{@iY6U)$IAyxx0mJozTZXLZo(n@)}qI4>GAh zRdWDDP)HY3vWis}7!FEbq6n%BpaURPkXW^13IP;M6oG*iKRhpPW%iTt!{_y+5?{!( zxw`GdJk&P`di`q#*cnUG>n;z&0rm~CV1N&WN(bgap?UBs@-77&Bdg+qZ9|+u9}Flt zI>H=kUPb9#BfCLai$&j`5iz7g1)2%qjd1`%ar3!7@aKTSU{n!aDJ)h+&|u4GEOMiW z+OrzNQ@Dpj+U3cJ2Eh{m3Tv$E+$+=SZGLO4SXgs5|Kw1<)wYfxk&EsUfd@x+_r}MO zN6tOu2Rw(5p8zbRfVeb|trJe$icj7!?N=X_(sz0L?&!iO4)r zE_l=o%_G3a@f5@0#?&FW2Aa%Il+eKC2pT5mXRZyebF%{X<1_n;Joskp{SO^q*wAuu zT2f?ebX>Hm^)eU88w&^<)|SK4vc$EAkB^ug^06Gxs08kXyfUEwfH6oof{1Jo>zizf zF%oNwktTyt#cxgQ;+;D4w@F|189Mnuvg42q28rEG`hX`Ub@%xGG%q=ZpyPNCg6sf7 z=@g5E2Qqnr>XvC$5E1NxAA@8*3p@!4L|SV>z*7JofO$U zIRV-z(PMmy@O?7dUw8q=qJZN?0G~ll8qortMu0qMU~fl+PcDl8b9=Fx;=Znn}`$GRjiq@P7c1wH*^%NFV3aHe`Q#pjzz@rDFL`v z7u;{+d?UsT3o{Mhdh*rqS>Q^ch$Ca;gUmTGEI7H5v-^0ORbMM zVrCrEWSvI9TjG(3aLI`KDJ}uwF$NZ-po#zj2}r8Itwql0AyfkSrx!79y5Kdj@&3gJ z?i4vAIZkA~y!aMem3U|eQZJ&BvJnY!d|9Mu2@o?*Vy2i-TexLo&e?!jvd*jBx188>1{K7I+@ zkb#@(Myd)KPX#_HO5Q{PB_ulF2=G26WFaWc&#w_o3S@J8fqgZGjBoSm=mTv%v*Vh~ zaeQTSeZ4(WyacitOH>CgBlD(!QNu!E0I1?-U}7Ph>yju90SygaDXd>83^y(_w??rj zy?`9}N5YiBNOLU*z8!OF;Q1>j=TCZf)V<-gp0@=v%_MweWG@Zx1*(n54eLe(NyRk_ zHyL0AQC0y)P$?{?j+`>=Ti6D^D2S1O4RCzqzDsv}^ym7Yrxob->hA5A2idZ^GGP@~ zpU?3@@dp97Q;Mp=%7Xs@Ul7OOWcHT*=d~YE zx#-1tky*W_2K2;SQ-I)*rw}+9&{0uiQQ8Nqi&qgq1X@ufSQJTt+2wcV4b(}%n%=*8 z>e&l9>i3AO^USis{Gi;nk5ZaQkhL;NeSAv_R52j*2z*1yQY@-!;GOA^Q~;t$O%sx* zD2mnH{v}Q76S$&{4qqNT;mF;Z5xJLc*mk7NDcko#8aC5M4Xvxe4BVso3Mmv>gXyKI zF66sV0fY!5Lih+FB4;hhGVY1gn41wV@{tG(^=$cYNa=0A=BoJVcVgvA-aQ4gT6E+4 z#QN45#v!6nF1QM!3!Kx1JU7mQkfy#z)8FjdC!(ph3Px>5^O$=E<< z7j;^<V@s+X^`@*j2CwT*43PwwTrWpdGQ%E7!-d2t16YXh`vkuooEQ z0tE^I5Ey1W1ejGA+6^!j%-;e%<@U)m5weeUjDc!MMfDCDxEq zK%7QbSQ%Ln2EcwSGmq+D7{U@@JWy)5do_Qo(zwF{)Wh1*$1C0S^d0iU-G(z03l3ds zTRp&cBu0V9_owOPz*D7(eU4~03Jj3$KnxyA3o472RTP8}z_m-j>H>s>68fOQ#LdNE z^AbxcRQj9WrVe>8IorzW1H$)@wpII)!Of69ziQ=rW8$L>q?f5C6y))c!$A~;7F>wr zAxw`OSVnY=)&a^vzq2-N}5P~;rwKxyFa~hY40HBbdvToYVy9pK?IRgcL5xgfFjwgx( zJ^o>a*KWqcj^CYAaPY6!o&9HhG+^oFO}5=RDM3Qd`DqD6_z0LH{z{$K^CZ>STK=ug zi_aHozW-In9h*zqEm|gRQj<{NXTF4hIw1e=B2s@RBHYD9){90*RSzxj48x-!3&0gh zbpf{y!xzb51rQ?KDU?o-AS{Z0GawWZK**d*-`!%bgYCRE=-rOJdv0#4X)A-Rbc^g6 z?~TOa0g95;3K@S!7%%fgE2eEMpthI=>k4vaPtbr|6y>}Jvk30S;%4aL(jwAuFZAr-hT=@R9Qbq8`B3ssNQnzNSy6`|D z#fbes>QCvWn%e?5uO94Cz$V^%W4P4v-G)$hq+`Mo~B`ua3`c zY86v>jsF@~in+xeBhZHA9O6Hy3IkSv)YBvAhA0V?2~B)f#59S^M?}ns0S5}lvit98 zm*QT_`F8WXjkliu;(Nyx8|*>L4f?Y2Pig5M)Fx9 z7a%JBQNe~vBUsOF6mKHVg4~9PY#sX$G9*Vp((J}%bM`=i5=Uo59=>z5#fFccUER@^ znFr>5MUn9T734Fpsm7nCSTLy6=kay{3&C{*!HFO-dFuk^GYfz-rHDLAqmW(H8B$&z zG#2+Y#+t02%X$A=c4=90ZPTQDqvj^ha-6iW&cuGoM8lC#p-;JSP-3V;W;K{cJc955 z(jxOOGq}ESryxH7unzKel25sj7-67Z3oqCA^Xvnnep)^`SBb&#n|s)bY(7*fECRG1-fp&&~D%L8E{8f%NlkVvukok{@MDB27VEXx55l!p2XR>z%qA+Fpr_1Hr< z`yQhl`)n?8ce6Wo3gS6v?i{VBKz4DiJp@RGK(~MdGfX%<`arpLn4)k>kW)tX1=U#y zXsHMs`^G_7i9Rr7wfo{Gw>A}OkhjXM!&fGLaPN#Q|JrXC0`2NMR_TA`(+(sZhj@j8 zkQJ^oQb0ozU7D)EY*v8`;Yq-r1&3cVgS&re$8eknf}GBR{({G}}+M z&RSr$2pAi9e({N3Gx6g~838iE0jmOCAjv#=6aY0T5DJ-Qk=Mc#Q7ACrNwJD2&2X?v z;;zP0_t{Zk;xkn~?y_g$cRh~JO1fmrr6R1#DM{#bt*R*=@>7KQBScPsV}zUsyi$ZK z@njOK<1RpTAu0plwvrGgG?=6#{7NbwQpXbIMaG(Jie;}nH)HDToh9^MW;nJ0!%cQ$FO+ytt9+RY}sOI>b2?eXLm3gF1n2f1l}I>FA?3pbNjhR@6X*5KdDUjBKG z=Lg%BoR6G{5v#H0KqELR10M&lEluL#NbO1ho#WLiB>eEolZ0Afyr&dA1tVk=PaWMGtU}J+Sgo|41X0hd2jdSuVuLXa1o7AjqwSv z5HoN?gx2rgqKsU}n-{3wL~R(hK;k$s4jqX^ngk~iFJnOS18qYHS|q@-pM=SHvv;cB zIdv+RTzgN`Q8SJ#*k=o~fb*65kd4RiXYqUufw1t%_UIg~@Oauuz}s064H;Fj2&V(r zih3jjY}`7$LLHT1sRMC9G0y4l^b5qdlPDw9Duw;QjRN=fUIFvN31Q`|$D(+a+ zy}JR#RYU;%;O?68=%_(paR4Q7NHwy zA}l`-+?flIDAai1X(qU6noPk1)>J%{fHJQ@wvE>p1Id`O|H{4Ai-#Qfw)Cd_*M?2V zUNuWr+rQz#Y%^|>-Bgf91{9kDXcCV`!P7y)o46I21F|zfBw1|xsbsL)#|yqt;lhKK z)uJX%fAWjPVYlsub~4g7!Ax_cJO#0784HW(9)Ko@=Ad54jjCUugn=(WQ6`c~zP0wN ztPECq*b5O_gVH}YeRlO1Bb%?vVJkM7MndFdU>?ZZ83EFPy_bNI)Jd8d@3bIAi=>B! zECDS7l86Tn+zQR$l=?U}No26b6DAZ}I8;{(;C6mY-Xh_4s2878$JTsR>ii=U<3hT*@`~`*}DD98a zEX}`Q#9gam*1}}3U z6Ng|V6q(2=fO;W4paqOHENh?BnBz!~-+ygh?VE`oc5pxGUR}VJBVYz-qN9?fL^WE9 z^B}Hb{cIR=!~k=7@PrFW;mJc~2=u#XSQtP;;x!JWy@7=Q+LSzHhNno=4kx}CB;4?E zCr?}WP5yNoe`P1dCOYfAx7Kmg#$Ele2*2^bD*><|{0uZC%H2^A4?K|z0Z~M0bOwbz z_$`Gj5c1y0DGF+8c;9&8)fjAY+NE{V#zMyh>tZ6<8#@}TUF-P42B&g#-$&*BelfD_(TL8<1~-)V^`znuBnqC_;g0VG$6UK zkzjbjWfzcmL&T9*NXF0&Cjj_wM4)*jE0l-PK}dwqF?Zq3`6YjN>ej*JI9F4>^t?|V z*!H*Ly;0G|qwVB}fVGd5lZmAV=1Ha#_8XJ~N3wAW15=JX*a9m9xJ#Tt9S0S;3=V;` zOeleU;!EVgu&C(3Rwb>j4%3uA59!90|EnAcuBSWcc6F%Ky!Tj1})kb|p^ zWCjaJDAI5gC%W+v8?aO~g4Mc+8hCQ%)2>A-wZJpm_SF(IhOV6b`RYdRotSU#8kC@L8KRHQ9NiYL0`kO;#E{;AfW+}6~NdWQ2BWBhXWdf z!Gm(?#)D~$HsebDxvAle)U;dZ%EaK&aCpgRbLE>&8y7ckaZ7JYoP&sKU ze1K#@*)(2%Bk)!r6QF+3e*oeF5rVx64WyzDPXm<9S8|d8Eu3Hc&zJt_w)gtI&QFfd zuAS&{_4BP&&ec!`M%>7)PC> z4m>m--BXZYL}3ySr30;xrYpDgbd;*6N~=j3}~FArE-dfml4>oy(<*%L0-oDBG| zxjR|dE5!M2k-kc`L2vDS^8P#T?EkiOg^=%!0k#dS^3U>4Aixzhg^I)gZGkDEJaTJbh*+Fz@CHt(7ZoBqpvL7YWy z{xJI6Ts0e3?Edb9g!KdB?c83OxM)V^*@w*`&@T8>S=JTA*{j-w_N{w#+&ON;h!x-V zb`P?nS4m*&U(yrAS*{6n?Puoxk4&y?$KGl+d7hoU$ap>mgur?Sg4je^-4VnY)4xmE zWxthsH(WXJ@w5xwTc5XKNjwJc#zY`R9K%5I(T|DACHRr|4E5BI3;Jt{HV>Wjn)V)BQ8(SSY@_N$5Z1S@3qsq4v z*8JN#fjE^Z@9r2gC%^jn3*nuV3Wcuz-{=G4)ZV)=;E8o*?!Nc-g`cN-E}sv%6B+O- zaTay~alSn!WqApib*^v1XOH-PL^eCemYAPu%T{rZDhx`d#s}o6j{xZ zz8_A#!-r++&)Yu#)~HBv%nEwKf3WL^bMe*@eK$_|`eM}rm7)jDob%0pr00h-{9=iL zzxFt=?(LjE-YMAh)bIbAjvvn3lV+5Ct=hm{uY4gFlzZ3g5i+Sr7sJi6ejm;bgHt+Q zt5~r>i&wuXclW`_Yj)p1N@Ge`8DW374&U1VSywy`b=lfdglJe zdVM&Lp4++SI)D^-q7XYrw(oP@F)-!i3-PdwZ-G@tWXrYiPmjH`83;KLGC#=eqG^tdhdUM}k*Ib?ORDu6&mk;NdndgO^EuXueM_OHI z(b9J3LS}jTf+#*w%90e=orcMoTyKXVN)y8W#y z4G$$upE@1?_LetyBS+#1dw?BG)%EawG-W>71GJoU$oz5Q4o2`;tXDgjD zt834gHhDh$F)3u0e~@FJC4D`dx5`Y#A^V!Q z{d8A&{W4#lc<*1|&BJ+P_{|eJw|Bl&t50kz$He{3|INKToN)tk->!5odCQ9MyyH(E z?YQSZ)5*j6EI)l@#ltsWFTZHAqg;?m548pE)zKm9BN-$J^JOb`d^l z^@asgwp?B7bC!4UaO%r$eEGq*L)t#Eb%s3q4iZg{hXzOMH5cb9xMuU}SK0G7O1)3839 zZ`PR5`7G++;Z&bp@qFVibKiQhc%Sg%&y`*N@9y5=?7QRn7go=ZFLJ{Y`n(l4GF-eNzqJ9fsu zt6PWj)BI1gXx;k587ZgMj*lpHwbQ?`SBG;+Oy}gUVw_&FM5VXty>xQKmo`8?B#oHb zWFw&827EtKhy1rXbvW<0=|eXt%%b%7cEURc*In3UlS6-d4*w5?H`{$WoHZEdqM@C( zjLElu{K29H8nFMBE*(x+>mAI<=B?XKIXq=Zk4Z;0yNdLm>(SvXsisVNHvjaoqsoW% z%l=c*v39OjBBlS64js<3``1S#yj-Kfd%2_6SNLr=|L^V3;f&NLO$X)yz zqWrY~Mt2Tpxx96ve(x~q(w%xGmVeT8V^zDVHk7?NobAh=md=-N{rvOKHgxVO_xa6# zUuO=dtMq24=ZThY4O>~P?n}>gu4)JJnE|N8^2v;Bqxneu-|Nfae0IdmUuI+>i*rQ62xrXGKR2=bWVlN9=7k_sQH$=^`s8JSc4U7D&V7KhuxH zIecK^$coF>{yz5DzOx6uJ5fK2MK+7NaX3%ToH*y@amUttsL$TjX=4%A?q!u`ku6Ec z!d@KCxtHqB4g2M-k$Y-=wlz<;g?axAoj9C(rsl0P<%;HU6rkSTmb>BeS*#L_P(fDo z;c)KDf9TyGx>c%hrcsUC6-%kB?5c!NcHwZ2D7JegU!+3KGD$5C6w2SZK**un)>Dq5 z=)vLqc1+^a8x~!@UAEgxgSw7!LfK*0Ymoe%QNQF=$JdrJXmN z;}-SLIcfF0%g0{%=<10QU-NdsM?QJ+>5CBe-EdZ@fA~~?dfWe(+bHSAYJ&Abxc9sJH75`&%aEKiy-)8S~rliG0-S zb*S?PE3TYa_#HbZ3@`cm`?05Kk%9avT4Z?seuoX`LWae+}n`4r-+)_ba}BYh|$?bDY|}=HK33!})!-h5K5s{PVYK^R%-ElLmfg_dSMx zNpB73;d==iHVqtbF~`b%pMKNshrxD`c0S0o)es={wM1W!fvoDR;T$n}UZM8C<^Db8 z=QqC`c691@zu44jk93?6Yz4I}cBrCl|g zF}u2qdv0vK_N8t_(=qqE-1$4oJK#|{D@2g`t5EdRaK7EEXw%gv^o~!?ty%q@Ir+Y| zn@jpzQ+bXdF`c&=k1^hf?zzUZ8314|J{BX&h8&qe`axoRZqJT zpE`W5^e?sS0AOJsW%Ry%`Zfr7-Zaa)X*kc6+VWKER`1WPIp($=>=wIGT!&!Qp zJag>}8>!W6uhjFgKW)i^ zmG*bPNVC#e)kDL1ZqLWB9h}rutaPjMm@&6s?qV}glaYoAW;v@mXgCuhde+!-xtFBae{cGMXVlALuM!=6*&^g0d zckqfKCGQOz)1Y^qhIuY@A7jVNH!yXny)x1^z912dEa{u!9B_JQ?NbGJUvL%}kZsz@ z?`9vhVdg(9I-iAIGn^}u=d2l$?N0x4r#R0C^-^a3mwIM6>+V{;DEkxF?!0_){jKeN z*q5#|&rhg6-a%TeD-up^47i_ph8TWY?2=Xvc}r^~-R6<*K@-SLgn5%W}Os z^H8bkOYByiJe2p;{uaKYndRLwoNdRi-p!Q0>e@N7%&MGU-#%|AGD_6SKi4b6dFkg@ z*DoKQT&UL3^7mZsr7zjZpkx5k`lmZ(IN81>-zfTOoui$Kcd2@pa*eTbWt$rPALx_e z{POn+V~!Q6-~WZ)<8r@!iP>Z46%H5}|NAZ(&TrP&>AF2fy`D{eY}aM`vOL%9l>L%* z4?eb52L>a&o;5u(oS#%V)%;kw%AGG#bMDDO@5d)3p(T(vpvh75#CWr5b#p1HTe#h# zAzunCCO%es&qbex$rcP<6p;Py4OLSoep?gzx%?0_D`QF9M%8w z#WU|&%T6AYL_;Gfp*F!iBw)qECR%@IwtX>vHx^A7xhQeWcg~bgzm{O7=!~B@a(&aZzZ`q7V({X#>fuO@3;PmP=UaUQ#w^O|JUn zE;)}io33@3_T8KxPlSI8I+cBNDyN}S*`QN7J)NJjl-s6+`=%E2H4_qIq80QojgKRI zdwq;Q$Msy4QsHzJp~x$3|IEGP?eMMPnQ{IYG{cwu#U*#{`7!@fg>y;WN^Ebq{chR& zwVkay-fYzUn>=fGz5IRrD~-1#XP>NsDW{6x~~_>7Uq1l z^RT>&nosJL-17FtrSB`}!#~Mnl4h`kCMD$5k&7;;VIVmAIDRpm-KZYpF$wa3Nd$!F}f zU&8{&hqFnx4oo#WvgK%dW@5pixtZl`-SK0x&$5{6hE{S~Vp?AI{|#PDir&n>e_BGK z)+;(b*;_k4HX%C36jM~hho5}xZ%&F$AgTwW4YksUOxL1H+6`M|BS>|9Bwid3bM|X} zBS-t52Zmf7_~LJ|jTG_9S|TrRpU#mzJS;ps9}^Z{uqpn(F8TlW__zeQcORXLLHE7B z(S5r`_l@Zl)i)+aN^y3KIsahoH|3|T4lg2IE>U9endEf83R^@hiKs*gJ^dol#xgQV z^7imXlKnuByhx%@y4KN<1tJ3BtbgK_PAv}gd~e64!k#W0J@+)**v^qXL+`8FljT*5&JZuQ{;r zN}X+b98^r|zl+r3$StBeCItPqTRc=p@UKW~=vJv2Gkf$XLj1!`ig27a}I;L6N zYw>lWy-Ga3K~u@vd&7t8(J>%YFl=vlz_O5?@-Ip7mYM(FS|>uJo02fi_o z##r?6ZW0}*HH=eYl2r}%4_S9_zObpjf1~wD^A|IHe_>6asg$I2!<0?-tzpq=>5U=h zzKN6^hwB!z+Y&Ij?!96o$+I>}H?$HO0*uWEPo_{|NUw%cqEVJq_T~Y;!;YRKaVD3d;*aBy@0hQ zr?pY)-;HWpw`Hq_&0g#3Vn#sLYbHi{^Nfy_`ZUm_gjUggwXo5#(Q*C{qwvH-a*UK1 zos?2AY;6El|M@59#V0aN{r|QV7CYd`rxoM(QmNvEp{ceplAQN$;Uj#PZo#m9X?0Gp zsxb;r6L2f)@rkjKjoQRTCpGtdO0SmFDxImmZ;AA2l)7kvD6C`W^0_8mge|?B}_Yk8NRQut`y&h$Zkoo zF$KduOZ&Cvzb2QepMh5R6r=e~E0aJb-?$)?Z;Vuhr!Wcd8d{#m?X6t+%D&~h&f5J@8F<}fnPmc|m6F^x9zZ{M!d zuw~uat(&#%FgG*eiCWKOxB_bU`w#h7hTOF$tGp}iD*M+q!_75B6a!7pGFj*IVPN01Iu_M-_K5Pw`CQ3b;eKjv0hMJxe|-$$c%JWLl{ zMv0m1Quc`%#H}Vl#?)w7nwk;Ig5j-svpr7i$8?yDaDf)!lK$nGl|dm}x8QI?|wJ?kKr31!MtyL<&4Pv7W!Er4i%HsE&V*U=AW2@DmP;1=$bu$ zjKBY9lkm-tIggA9-VOasNWv5m+VoR-e#eB43v29t?vs@zJ6CS#eI|U%V@~BE6#s#_ zw8C7FMN2=I__aS@411<_9_`M9i0CO{ryqMR525%EoD(Azo9>+Ktv~DZ(CD5A`$p6( z-fZ`mn;-j}WJE4hH=CLiZyfXHkgJxOHvuRT%{u7gx4aqiU4f?$%xp3xWx}NA^JRXs zfs%2;(^CqScC~m<%lEYfU$iQgx9ayL`+fdJhmEc8Wv*QOM1g2GS4yHI!>h z#0lZPSPR~~Q?pL_s&(E{efso$D_>dkcINk)!Fa}`ehHB-O6A_CIMPg@%hWz+>yD~1 z^zPQGhZj_htP;KQ1g-->h*W>9jYJi>NnqaOU?K*@R39-Uy*A1%0u_ zf6L8xkDPSf6b@&vclqfeA9%x8JxYi&AoF7gxi;^AYXdB{-Evh*1e3MJ=0<0P%noU6Zk)cM>^V+-ze z&WPCGG54c6&yY|kZ&*{c1uyuV_YZ_ej9521&kqBCY_)LzBm5V#;OVLS93n|aGFCHi zI3Bp_b*&H9DP8o|qRW>D&Z^M$&di)wJ3cWid|hUO?x$5bJ*`lkRR4Dd@$a=J(53VG zm#SXh@cz*DJ9CNB`~jJtR6ntx3HXRz4$%%EF^@;rh<#%*UBfl`)l)r{BkQh}$vwN+ zAU%9>W~*+DH*86B{MC^1=~D$3t1oWWzK;0)-HOe=xYo5~vq6o+mpz)*AQGWEi{`5O zzN)e2zwWE7-|IavI< zS{=t!c$^guA`x1-C7Q}&b;zZ|FTa!P=>>;7J);eLsr}23Be$vJL*;osH12G%U8{7> zK{>{Dxh?axBP*<4=E>aid<3~r&4T2;z0m|LZK#Tt{*Try@}4={sB4M-o?oVKD3+(i zM{$|YB9)w>Int5~RYW88Yg3IAYhKnw#vGoK*!OteH{V_|Nk6u15Fy0sCi8%)%yA}#>e>H*2LPjYBs%Hi#(G)@3MLMzKSbPh40Ta z37AU1^5f8d5F)jo%Em*cX-)6ktQ^#Y7N>rrPvy}Piq4Y4cl^ESnRK`PbGzqL^DWaI zty~lKUq5_f8vTWM^JI~-_3DK0{;M;42kNtMH^!d-k{M`q--RAr$`s)fl{lMpnEwpnkv#M@un&GkxEMrr#I+W9XkHpZ;v{ zSN3EYPz+#AW?_NWF+b3aKOVVr)SV_1e)%$dd!|~@B1>GUhm+kABK2;4yQdDS z0La@o!7Kph*DD(JMb_L-ovQAc{?XH`E`0OHiVv@Bi3;EMcyoIk>Ys&riinaV1y8Ro zl8`CQRCp!GVBeu$*Rr#^M;a#z6Db3D(8M--{Txg-mf@#IXJp$bF#zS{7%eEVfhVr}h9M^2ZHDD)YV7`x$z*m9YN zA+0$OZy<4KR55a;xW%lCNx8pD9muDti5qCw|<@V-O6FRN4--- zSmU0Md7?1GtuaZ3DhzIM?{n0AyJ4|k>ZQmluQWMaB6)7vnWLMHxsrJp++sf?TA|if z#BfZ*{0H^w=j`9JkUp|(kxA>4UhDjL+A1BPP|Ho5fK|t&z&BW zCvn+_5%THPpFW=4WI*aph%PLQka@fopM;7&@&slr)MNJWPhjyTEoZx>2l?a$BR7rlp3hu< z$OJMZ6iUv0u!M$cu*V|wEH|Uu?+KThRXZ{*PXwEYv6e2m7)smVqg&JsMX1w_2wpDAyEA8_>$Q)W7 zz@OoHVSbp|<<6^$x#%X-jwQsu(b- z)Yo#ldQr0kCZ`E`kDSq6FXn#c_uhLVE4jCauL$P!F^KHQR=bMvN|4v1Md3{h zkMHFS^Zg*N2l>Hdlzc_FJa`m={E2oPq&zlF+rUUtI_B>kh@T@+Uz3Q~N(7`QwwDq? zs&eq|Dc*eYur8VB6iTF7j+G>rTJ}K0{aq+{p$EE)VtKnv`fhOFgMAiP=ZTfBu(dK5fhUIl@dgk ztVp!Nx;c(wuuUxQ&TvMl)B^YD+>~YcUMf(g?y<77avV8d;~Pg-y)yVv*)#gCn)Bgk zR+JdGPP;iq!G9!`Rw>y{-Y0WO94(2uBI>NnW;h>0CvYZxT&0SPn6%*93+r6+D-Gst z4E_GVW|ShJ3!=Tg{SjQ0%IM%+({+L71UIF)6p^J_94DTmbeg8!lq6;>ajDb-XLWaD zerEWhsW10=X228DmWH6CKSYqQQS1`l2ndk1>GfTn&f79UCo44gh|TOVmsCh zzP|B5=ZbA^94ngRnSMVwzRsdKm_08{rUg}3MaoUNWgKiySKYjeS2*Y_Sregvb;(63 zl*Td^(`0NAml4Zu+}+i^Z@qa}#h&Newl3kY)9&Rj=~^qa>5~{vfqd{TPSjii=XOa7 zIl}_0YYMGNIOl?!mt>A|1xbu`A6c{lW#{MW(yB$tzub;!k^9rwgfn9tcHgg*48wA&#>qI* zA}6OAjJFa6w<#m_Ufg-UT4}jox7O{eHCsR8Pg{?ulhyY%N`bEy3y?o4E+l_K%Hv~P zXt+c(xT+PM*93)fxm~JTaSIS{axXA4xz{9_;V81sx@f#am^zvm2^ekio4-tz_=KNE zN1S@|xv}EKe&Zcm|JGfm#w)&U#ZM9BL2xM{I<2vh9E}MGk&MU9%eaDVQl6>J!$#$zJAr6|-4U*cdV$I+FF!y2`mlo?;nBRb4I$ zFS{yki4!=y^$Htevt3Sw@ z$6Gb;IpN%g{aQHev()NC>hH>&~T1qA2K`TXsQ+IL4)NI4~0Qu_p6= z&JAe_TytvP>7u=#|8V4v`M=!yMMQURB15BHpD9DQ zhO**#H_gzR$_cPRRKe|rr5d<@gs_iJ&@Oau`t%TL%!+)=`@GzK z_nE|T4m+I&$$X-sle#6#k-8g}CgVv)Xg0zK3dARRIKxY&RY}k#K^8=cm)#H`TI4xR z#3W>0hWHCQ+*Y0wg2qM|V@+{C;QeCZmfn57?6l#TnGJtCo8NKRrZx?*I|F`nyU?SG zW;_yy$CXtV)IJTF&~=$nVN570&8RFmhgYD88pEre)Ugr1u_m7tK508MSB>Nj%Trcv zddoSyl4IxJy3BOzrJ;w11gwn?hzan5f&=MNaZL-XtRgz#(p{417F4Jg-sR#Y7YhyK z)Bh2~76Hk3p?mIVN;Zhx11uoCnrSX zO7Iw;BCJp7unCJvm3S8p1+=?H{4R-OL|LS03ERVIjKsNE+NH^w52)uEP*Nkvf3C~~+b}EhF5vRZ*V__L_ZbT;}m4f=@ zRaSRVf?HM7@NCiuWWD3iZ$2wmZDW$N96h|>wjaycx}k_|ZL)4Xn&tam>titUG(|HM z%VAY{6(RxSffrN;Q($zJa;Y#I<#XFQ zI_&hTe6*P3nl(I2lXaRB8Hv{s@P%0Lic63rfzcTVy9jX+VDU3DY#U3l_z#PZN+AF9 z8g98(>R9Kyx2iUM&}n7KDUKU|OC4HYmG*?i_@46htv#aCE*kd}2meYojw&axlny^d zqhyNVc}3?$L8cx?CU6FY=a;{r7MeP+?binzeO2nek>AMNBKiWPYj|_zRf>OR|K$cV!aVRMdWO3a zup76s_rAO#e(RouE_@m8;_qKNZreN`kK{jnatHBf3AdJj2s=)?EFsFp%9NX!JCcqN z5*#p1V})Q#E`UtnKJ436WZGA=dX(?IHPGgo zK!TEd@N9x918qU>*A$;q}w+6m|5G^VYds=BLyop6Xu(VhHHe^D3NtD2-7jL zhi+rm9?d>1MK3&^9X zs>WlfIk;pyT=eM3S#TPh6^7MW*b*`&>ok%`tj4np??SQ(SsGUMC-MB_jindOrC+;l<%4U9 zOqW*gj~sPr=Lb*Ow(HU`ok1kR_k5hCUUW*J-K+~STUv4<9_W@uMc`PKqck_KbMS~s zTp|r3!AB!-*`v4pw4STpzWn-uo6m&3Q@nlX62nG_KuS)EhpNQGiOGqYZ!SPrXvode z#0{oYEIr@|BtOjPtO9ES({mBUY2`f31Dc*%;H)l|a-U*`#Wm=@De>IOvXg%eouVOp z_~yasNY1FT3w9{DmypY*;O#I7gaSQ+yBQ7x334%(L>eI>V;%u_@aZaN*RAb8Z(jvx z^8=Uj7tQOi(_-(j?CcD;0!RaeqoC}VVa79Dlsm|n459YBCrT) zv{D8>!lFrKieG8Aap!?IxDC&a__%!N_$Tn>7|jwSh>t)6s>0wzGstDYLq=%Sg>(QC z0dPv8u_&O9f^7#MiNN(Q(kSt2`(;flzW714uY2SfamZn(v-xAm5>RNqLqKx~`nx%m zk~onCM1oj!oJLF`o`yY&Ofh2X2xSCG5QYcx{gv|W#LIBPoo?h>`}poeE?QviA&p1`4JX9UM?ok3z1Kqhb=lN`Twn)pVwgP+%{GUe{iE1t29(|MFJ>*HM)Ss=R?_~5Sv@g86*?Q1*b=%Slrj`eKWAWxl|ADpp;fg(N5NViTC zXAqQ?H$D(l5bIIEnI^N)VD#(OaqWla5fyplqzyL2GmO|?qB?tn=C=`5PR+B~K z@Hm~La62Hz<>%KL9}&Rw(Mt=(@gXavcFKa)|0F=)i1Mb94 zYaDcn3N69|8b@iA3ZEC&545I;P&>ly$cUN3Wr4Jb8|?GNR@}Q&@(1~Yt$D}YyZFJ4 z(TlU}wk2z0eXViH1fp7zszz#*6M+_>I3Oq)Nk!veq{wIr{yK16xPWEo%Jc*RkKO^t zfOk4PQ*G;mLQ~vFcD5Vp_$6zWI+J$hh7*RxC-`>|Pg0}fs$ zQ6{vkNtE9e_5%`4PYw<-h?sJIxMt2B5At2=k#km$C7re->uVR&npTHc-)XI*^_d8Q zZx(=LAitsLa3_I25oMa>vVoZv0Qy5HT>=EcrBbY4r3aD-oWrqF35VNxr(N!O{fl<5 zzc%ndmO1t51gVFXf}5IwFZVT zS~LR1X=ll)D>rm2FBWT=t4*H8*}PfhxL99|eNWQ*0MC$|KpMRHjxe(9q9Eb)BKIq5 z6p!EvjjKn7Zp1`5U^`qS;_?t8fnk+$Wjn9X`sI0gY_D-2_w3$hVV2ni)|Oj5Sf340 zn@^0ODG@J3M36*obso@5idK-rBuQh$wP1OwoQkMchBN}@sQ2ym-BxVfxV}-HT5C2G zyI8_;Dr*k4^~Jtj@U1@XCj|RxXd@c-Jrcxj6oy)Ka;Lt2kG#o7= zd4y<*-)zCiK>4UNXhQEny(a8Aym8Mfo$9>ruysN+!ATlV409rn?dd{bX3TsGl5TPKvn zlhb1*gKU`8X3t&&-R1$T2X#Ii;17UmD3q*Xl?4VM8ANdj&KNK_U?+j{u`HPY3MQ_t zyOfAmkA`onIMj2fOYT+Kuf2S5z)r_Oo1y~s4T4_(ngIySM$!Ue;tYpEB!~qAfDBYR zPgIAGvDc~5%Iu~pk;spAZJ2@H{4Kmccid?mO_HY-SzioEyDd!!#`1Q5}$`*^h zKOdsFVkQWl zAdrjnFvl}0L3RUdfGR(L6(#64gp7z=L!`$T2_Yb89z$u(=7w#Lz138zckts~{5!b? z#~~XG5*Z+!D-)BtdwhSI)&oO;@Vp11CURyd7FjT4-UZby(<)?w=ingXY_UL%lIV2M zV4Nv3*krSJqxxTOZ}!sn?_R(C!&N6nmm*K}QryhTE-4sNj_}8dWsFlckQtUSp)m<9up)tM;8;hn$h0ow&4S_DIJr zn>==tVu4~t4#oci;SRO|?plkN37%c%H{%kd!%>?4jfHo4Y4<66891d zoGBnIZU(m`B62Q?(%_S7a4cZJ05#~wv2kk@i+V0V*l=LXk+_X8(%fa?K+A3&F1-3# zwR}Go%{RE>NL$$TOq8aPJTnYX56lppWjBHVDh@O}9pu7ASw${frLdSfQkpPPpoM(? zzmYc!uw9RC|8CZ;6z@C5rl-{A!#B6gYWi&Iej z!$4WGsM|u}mc=PX2I}`lOB`B(j!zBmHumzVph`#mxP^boM7HM74PKT)GUZSYCGzWGjkA=*AlX`Fd$Ws>^% zmK3E5sLMg@5=cvmMX4SPY#ow{DhpC5iQFA3?RB?*Nt5~nu4uDspKXr#bJev`&=o+iYW^sWaWw>- zI_6&CUrzr}F<(pN!z-g2EUIJs)WJ?8FnEvePt!_)O-z#u7I8il10fKM=p=LjGy!UI zP`pK8v@QXw3up;~k_6%7v z$QDM6>_fKf5wb)K!pIUa{GRvRrA5sAj@wMlKf3qmynW6&?|I+P@_e3$XK#%h2NIar z*u=;i@Z^A6NMxw9A#^F2$%Pwmv_lns3!4r9CV90oC2jr!^Qc`D6+2n2B03kjAZXTm z%>{=T-<$d?aA=X4L8=wmSAb@*DLbGk7C3ou3LvPl<2mp^Wg$g)_L)%_uSe(inDM{k zCuA)=8JU#qI93#He3a#j$#hey{6$DCYWgZ|g>bKVSflL$|FD+|YpP0+0!4hEI8(Xu$phx&Bq@hI?vvY z&$mPKB;W{KsWou0@F!6wlK>ckD3U9uHF5zO^h+4Xvd}=9OBwYl)oWEv{|nt7bWrcS zGtP!pUP0n*HCCyz!pj3}+!&$urzOMv>Lby(oc7=W5WsW^stH*1xajue(M~xBJ3A@1 zae%~uC33Ju&ae3S8pqRKW3TJEvuZP^r%$U@9edk-p70@S=!$mMXkU10yNxgX%Qn82 zK}t)NLXNL!`xGD$7K`L`PNN2e#u_vhpijjbTRYYgw5B7GgS0Q&*D=eBo{p(bPuJMp z%494jeY4s&t(&ob=PfmS*t_+NWxm0R!e(-j-V)kjLvfHe*dr}16(Gh$XoMXcC<2hh z*kc{R7GonrvnRSb3QmTR^kj{ZYdwAsu7A7hh!ssvta{**+=x{%66sSQmV$y&iq`(r zCD9S)SoL@9?dt65ik>UXVr0e8fC4xuTuv?)I}|7R5{Wf%!(uxJ8?buV35tw?jl0ko z8k;wrI`;bNs?GwJ_^Z4ByxOEWD?AEQxC0v+1)Zy`fLkj>tSw~z(~=$>uelQ3Fe{#= zSQaF*7TVCFfQ>*7iWKlZf#`(~o(80&01-ga6fL&ac32)|f=@=VEHsMdNp*j2K=wa^Bc2_jUTcw50k^sM-r&?lTk!`!ADD_n4&N= zv}U-ws2mD`??E>b@R4)_1Um;BP0>Gse3=aztzf@6 z&Yh#O-|3$xtPsCi#LCTs@t{GWl&=V@CCK#Rp)x;pc^-SDc@XDg;@SY0hb#}}rGfh_ z0Bn^jaj-+f3+gU%KBa;vaxV63oU1YQ3~_rs{E*d^oQ*9kt1o?O$0|Mj+54=ERJ8XP zksz8g!5CmKLE;R1Ir0}gSc?>f0~cs9bc~=7gKp(Qk4ig4k^4J0t)hA5YHtJ9dK>Rf zbj$B;e3HfFprMwpDnKUjk*kq-(0-tfCCx~~BgIC6ViuetP@ZsY}!%~5+E%%g7@^Yoq0?{7K%&f%Ux8Sh*6a^|l3Iu{P8K*C>d1Aqfnn(m;5)*dCZHo**_0yS&W! z7K{1%IWsLGIE>~cy|FawFB1S3goZIBvyg8=BcDt{0Ts5GwPIVeMIZnyY%DWEQIq@D zo3?qxv)MV4jlP{MhwV9Je1e6^^c7Qrwc1|>yqK`y6CXK2v`abwzl0e85;HJbpr?|L zMh$Gl@FN@m=Ccz3fxri8qHX{zTiS%+d3WDK{Df$J+jXk+O^zKWv8H$#2gHewzZF0P z)G=0AjaDj0gMbQzk%3AOThMf*kk)QcM&ScWV-Lrfp1>2LSI<_8l@(4rSnxzBQ3xae zh5|1^i$VBRgi>gXmKPZcoA9SYX=X@N^ud|A4gIYqa2o|Zw45-5705}qRBF9UJu{W= zJN4>qg|?U$n6;<~(b5_hNjM;ccqkR{WfU-B2Luk^-T~>jV#8s6;B<|-ORC?sH##D; z9o6~ad~1G{6xOu?%0oqSsBnJ^eWZVhD@0ZkUZ6*`l#QM)$yFTXtM>WQ}5wfMo0s`-UhU=3#a%ZLxDRTVn^0i4$ben1r1@X^i% z%o{otz%Ph09Ks~jV61I`=>}sTI<4sRC=xm9$0-{BEuK;9$>df}_ajZNI=)=&+m$uA z_Af7lNgwI1frm7x7+@wKBY8CQjdGC)2w{-kp`ipMaiCm~fR3%`B`{f}BGGF*-oun1 z=AINMl7Dkf@Wy1$rjsNs)h zzR?#;A915!Sq{n^vYz>Y!Ky3a}kQCd3B&6~!CaVGWnt@+5X(hO-Ct9;SvE?UzWL1B8gi z7vzv^~j$x0r!qVExi%!xuFh3>7FZ%kaq2al^GmSBO{wBqUx% z2QbQjMo@~K2fm%wZ0#5&SiEQ9?0SpM)gJ9>e6U)HkOs3Bp!eMd zs7Z+f&{&ATHS}$?<&ohdRgtg5StXZBw3@Q67QO#)scldY!tJ*M zNXQy63jp%L?1rdOCgvh2mr2nJj((!TW;5iipU!=k?k>0XP5Cv!fpf%VK<67KhPd1M z2VT@{K3qdgZWN4xo8y;@X8>;gys$#w3h(e{k z)A_ZaU)nsU877lMU+27T%qm@4Y=($~K|Dn10ScBha94>%0JOxw%gV4jn`)kPuxKVRguNxLCg{Eb3 zhFAlQ;|DN}pU?q290CNesUW5mQ3^#=2H^w3<)ERhwTO1HqLmWflMPBLI0bofed_4D zq%|G8XEeMyW_wbLpDG@e8K<+@SU#5_2W6%%J#Gg$5SoC|CJY8DYa|qrlace$GJvC8 zR8fEwLru&;gJIiE7GRLDJZOPM%?LO_TIYlKDK!l#jP9W z@j9ASo{L(`B;l~vsc2dbyjZ&LVAazcpqU_)en7A+!UTY;W+w$M7fD;-Gk_Qa>mWes zXiNjv4cvC0oDUBP=WD!-I`ThmHz3Ywr)w^rl-AHOl2x{u8eOa;|Al^xKy?5J%;pQpkx*E>=C^P1Y7`5&`ORR11RETVCWO+n`jGQ?93?0pXwU_-<6|(KH9%BFYws>Z?8a0cr1x98(Azpgkz1(S;!c3gz=-(W$Q|#C^K8Qbk@z zx2NjET^4WX^wQXPE+?sLd!ZWwZ-q}!Q@5_J&ZbbkWW>dU-U`u31yFcGaj=g9IRFKP zW2q1)jS~rJpia6kb$9wL@^HtHr+FToTQ?uD_wNox_qpIeWCAtg#P2?)zI`*AzuN4> znNuC+US?+!6+|I!rb}!j-VJ~4rp$lc-g?_?7A7&{L_N<{N^{%xT4B=2+$}dn_@9vY zAcxlGt148Cp0xeX`oA37z1_7O3w#jgSr?sk_!V8{{!+K#B$lk#rZ~U zLlaCB@8u=#yIfAiJ;;rE6C{q)rktque*XN^H%wcX`m_ImAK5+Ex*0B1`L%c05+8*7>wgvKEll36>m;X2QZNffL<`nY{ zCOaEMltEDs()i=?D^G4;ov}D_$iM;D8`LQ0k{)E6dyk5Bn`ZDQj~Fg&)N|O2f)|kQ;RmwyfcR0fTc~Z`atHoKl{JJV?FgkLTW~?CfUY zyZl2yQmB&6?)8NjhN<;v;RG~FN5Ek)84ohfap7REF++b_kQ^R=BgAGJn{^ehbV2a~ zaoxWt;z7)Mb8}ws-#_D5u5e|F=i*p4&!Xh(VTyLTHd*Qu8Ct@FC?=0+x#Nx5a*iZz z)3Uq1URf+w+P`5tb0L^TlY2ifO&`SPfsMHR$5IO&ob3-^{u=?=1B==dWkeeEWBw@c1* zdoXh7e?z!~$X4DBt(y45+m*BLzE~=JkX7oDB-1#jVSqBAYzLVgq2Ad@EFej}?J8RS z9$umBigpkydDJ4#&DiLV@!eaw#J5^jUL-q6lZ$ibM_Uhxiw|qlYUI_UYuW5aG#~pf z2zC%t?gQTXnhh`R=eB9}qD{4>?cD6XuVZPbXnP#>L87!>Zm3)SInE!%4lN6KkCmbS3(`+_UR8~m zFM3&zd3>WSuai2f)D!3$P@wi=>kWHqayEXSN*R9B{zcXy~?bDbGPN2F83hbYWck-G0xXyz^<tkuS6+NzlARbkTK$}^o4x%oD~GM`S!}V?@oDJgG_*7i29V<*XO~3Wv-rj2LX}D5R|swkH?Z~I zikbXRiE)thJHMq~TXf}cu*rlWJqJYoRBDA>hCHW;Y=h2SFtij0@mh6s`^o;(T-^?> zo}Tftew|WVTneQ`3?Rfo7VfSb6xFO}w+)j7dmk+{sa0-eI7q~*Ec;6Ro4&6u`n68n z_Ca?_&GXcz!nIS$Fd`fzz4k#_U}nhR{tc?yL{EP=vDB>B>St)JH;e=a8M`&Lc2kd2 zEhKZV{qS4UWtGZ>00%MNXuI*`@*^ATcS&p4tEs13sd@VhY`m1@H%QNqm1ovJGUcJ$ z#h|;bc&c9IMtp-*l8;Rr>{#Jo=+=kR_cfl_q14jJjQgOgziUEx14?g@jvZZQuOGE! z(UCDfhOd!q-1pxR-XPCCgS{%4s(&6iRxzOZx@SS97D^ULNf<}OL~8t zGtl39UG{`h^G+%AUFnE!5Yv-~e>cx-Irx3wn$bP}=Orz*C905ChIVT(oa6@SwX*j8 zhZW8qd=RG$soqe%pDmK+uN+pbNePGpt(c;x@yNV4J zs(&(tbn+P5CD1?;8|37Z%}I&!TeBwbz1@C#_-1nhg(bQhuM=k|9f1up=<3*O$v<|M zCw6(VeB1fhiH7n3bn``b8M^N}W-wYKGLXCmkv55M-s@bI+-eQSa~d{nxwCAGYmf;? zo45G&=T_N0U(ZXhPReXhHl;PlfqfCLvjm)hZr?DFmcF+BLZf~XuC zwVBe*{95=q7RX(u@3}xZqd*D+-XtZ;`YI6(@_xX0UXQv0p(x;G!2Gj`caO2OP_LY! z|3GbHEuld=^GV!{VFx0_~Vw@5;&B zY4x9t1N;>e3ormpqgSjcajdAFATk z+`#V|lj}8vTn0H=^WyTy-fddn>e>0FMKkGcw(nYLie-@S2FG{V)obniLr}l8IyHya zDs@<{j^0q)B4SXf400nP@Vn%=2QOQCn@<}#%PE1)%AxbB3j%4qUtMrW%PUElWeg^i zK?bE9tQY0BZDW@?%DGu}Vg+SFCWFWV+JCdH|I6H(MltWVj&U5swngDDnMMsPl0jz9 zzp1`>Cm^2+d<-g( zK^DYKt`zla?1L*E)<4Q`mmgy7w3nJV z2I;$Co0FMkocoXWZn&LZ8efx5=@1Ffe@+^MH0#p+!}}+y4>eM6nI~B9dCLYZQU-)E zh|ssC%5uf*O!bzhL)@!3SCu7M43ZIl)m}bn<6X%X()nym_QCQaib3*vI^4~_R-x+z zk>HRdylyTVR}M3})Fm;<<;icRwM!j%e){Dz$JcLFd$36^!_Wc=ixSifjA zgFL-8ZQw62*6f&i>6B{+Bgcg&S?+#1H4DA01?>DvISkS!GVJ`RPR&0|vzyy#^laCk zzRFG0D_<12Y5GH@A%;N|Z)PsC8$GHkH|wm$u0?g{vzfIZA=PUM7t^vp)rJZRf4vk2 zS!_PEc8Vg$*y#A5KOTK}_6VB*gc@3!LKq}@h_p@b_ZFjacQttSP3G~vWnBh?{9a+p zNv~ZW-{i!~Zl4XBdcW+5V34{erK@JvkJ;Ds(I3!Ty^+Aur*g%i87*ZW4egVIymS*2j0vIHGacrHzZ>qdgzxeUoPw7kVv(4%k(!412 z(E#!n#5F8_dgJ_`B6@^$b*p~YH{un`E#WUCIiyufPyB*pgm@*-xm&;KJNsK|ht{iN z*aBSt@-moIkU^y{h~n6Y1x@F58{F)v4^Q!aMBdlzd<7~O`kk-fP-zHXklEwvyZ@RY zA6jjLbEo++-`-$LvhdH!UXbK1d&73~U)7m#C9GPGxkVP6nq?8>MS*q&oT0B3y&%4u zI{j~J>)qd32bx^EWBSU44Q3I}Wl1FTpK29=GT2~}7vxs6114UcYc@E?WP3buT(iE6 z3tkY@L*i8lE%tSsW)#%yqB=9|KO=WR+FY>c@w%^S)$4#O#!HF&Pi*M8ip?RtOa&^+ zg>(T7CU!wG4t`tBsnYosTRle{>p8De9X5%g5^XL`sS7gj#V+4_!WycP;S-xR_xt03 z0q(S3qN3QHHk8l>xs$r7&)FzyxBO+e59lo!MlpXboK_Wua5D(0w<&v<1n2 z;gqyXkFBe7zB|Bo401?s#`iydo> zZ$IbBmU^1szrUkW1tTIFm7Fi?z*}7X`G{|)W~cp8aZ8|#DzB-mWd0(tLMH96{pua$ zr?Ao((_1NFB(9B;|L)~!$7HTdbA3iTXfz}nFY*ge$gKQ=f>k~vDQSkxf3z&nN+JRt z*=hi~E4$E7UbyF+IX?bU*?22zhY{H@DEsCmPxIQ}u3Y^lwzcy0!cFuRMAzqRRe39e z6;ju6K9ZoGkXh{$tWY@n<0^uJ1O1u7+aD$`zV zi@Vfs_?#N!61F*oZl=b+`&gH=Lu0+$Xn*WEgX#6VNxLZnp`sFmDJ7G-`bz_pK2Uep zUqDRyjgHq$b0Z$D@m!s|yUFIj21$rs_%S6&H z-l8CCE+l#j2J4d0{>lfZI&(9x1Wd`k-eh7dGgu0j6ZhG%JZ1h;?Kpay&L3$v$?1Qq z&R+cRbH;_1PHPS^!>s3Wc4>y`FQb|;g94TQqdibM4p9b52l)hfyNFOlhFuB;odo*P zHhQ%DUfp9|tB1ycwdAY)CZ1qMJI>{7){RCVtqqqu2y*xIk-MqfMGCnx&`&1S57P8o zhwJx$Ydf!I+_hKHRW8-u#0;{6tD7&H`Jo*|dp%x|?{!nT`UM23_2YD`devTCCC2B& z>u;>hnp~=Kh8brQm$SVDV~UJJr=5@gX#Gfv_*9c~eLYn`HbSADoBl`2H%Z{>OIH1@|2tXx?h-Q-NSu|um+sj zg?=e-Z|I!h=f5Ajb?9!*o*vI8Tx5RfHNl@T|1f#0nak9D2UbkI6uRqKRMGdls>#ZV z+ZHKC#TxIc%^PNQCX!iHvYHr$`KmaxHESUx;*ChZwP(EAq&ZE`S~~La==Q&SNN4`S zLO1+{;3_kht%+aa68Ub^zN{C+PvH#K{+u@~ZFze^#qFP2su}Lo!JjVI ze_IOalikQ@=D)NZaT2{;J-w*(=l^G~umar8hzL^Umg7_GC+wN@YSsTzZ~?-%eiL!uz+vlCU;LAyKer@(2%K;9kI;#f5tV!v-0`#+Zr<(oEX-1$cl-) O^M^K9G&17U?*D&NXm(cs literal 0 HcmV?d00001 diff --git a/.gradle/9.7.1/executionHistory/executionHistory.lock b/.gradle/9.7.1/executionHistory/executionHistory.lock new file mode 100644 index 0000000000000000000000000000000000000000..bb1ff082761efb72178ca8fb850c56d7a9a2ad18 GIT binary patch literal 17 UcmZRcbCP@0qX5=-3=qH$05%Q;QUCw| literal 0 HcmV?d00001 diff --git a/.gradle/9.7.1/fileChanges/last-build.bin b/.gradle/9.7.1/fileChanges/last-build.bin new file mode 100644 index 0000000000000000000000000000000000000000..f76dd238ade08917e6712764a16a22005a50573d GIT binary patch literal 1 IcmZPo000310RR91 literal 0 HcmV?d00001 diff --git a/.gradle/9.7.1/fileHashes/fileHashes.bin b/.gradle/9.7.1/fileHashes/fileHashes.bin new file mode 100644 index 0000000000000000000000000000000000000000..31713fc9102c2dec86c3eaad2de680d4eaf3a64c GIT binary patch literal 25997 zcmeI3c{Eks-@q?2&!R{Z(jeCmiio6=c^)#)QmG74WGt10%qfM05<*HNsmPELN+}`_ zLZyrq@$Pf?KKJ)~&h^Lp=eyo@_gc?pecGS%-TQma9`57Z?Ie;g-yABW{}iGB{7V0g zo&kCW=oz4AfSv(*2Iv`}XMmmodIsnjpl5)d0eS}L8K7r?o&kCW=oz4AfSv(*2L8Wf zzyb_}0}e)5ni2bjH;_b<;{v~=M(kA{AXg=b!+&<>#s7YAqH`?$KwaEb$SovsezJB& z(pa{_GsrEEx-DpC<-Yfi&+mGNpZg*q< z7L}nZkXuCKJYIL}Eu$_UNpyWqDxX!zO;7nX4!K?p&d>c}dQy8S8(m*V73Yat$)o!# zE-S(o>C zSwn6fkMr~nCBHTM+S?$vO~d)6J-VSHIphC8ZuAM~m+c=1Y^coChFpi8${nZLpU5bo z>)6NOJTruBTqXW-7i@3JOyw^4xXu_b!6m`Z|>#ijZ5SWnT%o zV=vBgo3kg^%e|d|+QAnQ+-4u%zACi0WYv|DlaSjo z;Jn&5Y4Wt)+;w!Fbez{-F?L#XV{{4R)`2*GeogU_cWuP^>#gLBz~b?C6xR>u zZ!@kIwpr?*f!xsm=k0vSwF3_JXuPgUIDcZAstoN(glk32b}lXXpwsqTF~=s6insO zv+6=QMr7FDNt?=tHGWoYsPu;1st4zNf=O*WM@m>Bw|k27fzAP2{gw6kh?A*&t~gon4k6m54`wd$oT=<%u!XvU}1`!S?pMaXz#t?z;Qx zd_{B}N1PAGID9oU9M*;0ascPwrKF2bTyKzp+(`)MBj-vBBy$_FwH*}GO;n!ae)inv z$p?_T+v5C(T82}Ezvz9)ZQXJHQ={SboWWyszjUeheQe#PVh$(QaM<3w8gD-q)u9pd z`{8YL9R(_nBP%xfiJ)~uzlzGojiZ%PR1d@Urd~K77itR`u`h0c+@S*JlVf7F{>^jd zs6CG7{%YxdT`{l#J^yCU@qd_=eO{M0evSvWH;lmfT;X@G*7SO`e(L_kIm3-J?=SrG zzzViEAHz9g>`U>A`a(9yjrnoTM6!6q`=$`hYr{UAvsRR>U-Fs_J%^5>IA=f8G&SX4 zISsF)L!GDW({JgSY2nDVmNy zy6qz5da^iQ==q^KVeFwMaPR?)k8j!ns;#|f%FD8lQ#B<0^4&q!^!s}C&Azw7) z1`;@z6I0y!hxOY{$n~lHlplODsLi*H33AgWyuI4^Vi`MC##@j(h~a!oaEndk@4jNl zcfY~8dQ+59bp2CT#B*@2IX-Lr$;*-zac-P%tC{`X9$h>Px#ew~YpLocDExDd5prGX zc(*U+%8jh}x(af=DZKr5&mZRNkB%a)AB*!HVt2!zD2$?6=8aO28|PEGlU3c-BbsPkwO@mCL#r=R9@&j3|1PRj zezj?TE#KN#@H&pAI5+Z?G23yU4b5NEi#Rv#s_5&QtK)<1oo?XV#AavPb^-GS$c>6{ zZnpMK_l1+PsJ%-j&drSvm~!<;_QLkA)OFH4!s>|l)C`*M4ug1m+pDh(?tc7<_I-U@ zoZBgXI20EU= zh061OEiE)%hSqtjn>hCh9NHX{>yO6e5QTH^7R#0cOV7Q7*SDR<`N0cM_Dz?Q(S3BI zuAja>Tk8C@XF^c_f8*`_ex$hA9ls+7xy>t_`b6*V1c&}ke(3NoZP+^@?CyZ?mt~2cC4uca;Hw5 zA0^fC@VGXkeO5Oe=fP{3r&EsSp#8|)9_RQ^5{U=h8+b)}P0s*51N02gGeFM(Jp=R% z&@({K06hcr4A3(`&j39G^bF85K+ga@1N02gGeFM(Jp=R%&@({K06hcr4A3(`&j39G z^bF85@P9u83ZRSYz<=0_A7XrNQjGh>s^gT`SMeG*ZC>+>L}JKxW%WRB+He+tMl%04 z62*#)dp0$a4_W6HG{@~4#|*J*Ui41T0?Z^B4ILUBwwtmND<*918(Jp7TU`v<=4CE> zKub^@Lgk+p9yb z{JDx5jzJu8SOgM@FBMdT7_vKKy7ny1=CAp@=bJ#_&#%D9x~CZIgtZ5~!rmPs7(UDi zk7^AA8_t9~(JKh@b9wQjX?Vz(*1QKZh_8uxB zhPCT!9wko+22<|L zrboH+i)%~m0!!~?BjX%27L?N(E!&1Zj-Du%o(f1WGEM%1-Sez>y)*lvK?2z(#+cBk zTf*)b<)(5paZsR(4ZLv$jGmb^XfQ6OHMYkZ9SZA++H;{myJ=J z)(F-Ra$kG%c+7rRZbg4pQP6^7yn!(o>}d^O#$?aMEDh%uhT9C>RrdLa8Lq6Z7ofq2 zz2`-^AL^M~;>~}{asY9|}hHl7EX9k$#K5m>LV=Slz> zyJ?LJTe?&ixKhDuU$*o?xmTMP|G z9oiWEeT>2_vhVhh*L?cEdO;%SKQ^PjB7@XSYxH~&A6Hj+d;V%$kt)N8#|jkV0E)pu zYZQc4M;;qu;9=Su?6)F658MYV#sC^QBlh+iA-m1h17n^CHyBx`1-;vD>8?sKZXc*sPqQB;y&qIy2TwPa-5`)NtT6C_iV)-F{cA}HF1~8nkAEvq^|xI_ z#(9cBB0Z!vw5&FfGY)H-O6-r1yK>7Le5;8edsT!RFtNhq3MxVjlh+c4B__AUEltOU zbVtu^!VL40^MVwC#NbG4c%EV_U^2}blt^Bs`QFI15cLXuV}Nv<)~IzGih2Hzc5C+y zA*0N^d~M7q_vG(FF^p*qXXfv##{b;v-Z5o9TxF#8fD*&%at1Q6x2Fl?sQq@LqF}=9 zaqya7{RWCRm5_mFcb?VfrET4zIoV>zVTFI(PV8obry-v(X-3NQc0p>l?kY$j3-ZoS!#vQ^TU;67$FEr9?p~0X| zYqaayvnKJ>`wp{5J3oBFjy+c(Mlyas>S>LeBOSMst1Mp17Uz~M{CpTZ-T$4dMT4|P z*@tu9Yp$!=O)G|qrJjBUzTfiS1_$;%6~f40q}&ekl=k9X`?hOy^LWD4zYW$gTEleh z!jzRebliHM3Vo@c3ui(Gc5P4?j6g+*Vg0v7-PXIIv2(h~SyaOx?8=m0A%l;X*2s!V zaJzME-ydD9_@}4!sc!2!@Fq1_+Yh-u11&*eEdv#y zS38RDs*_(m;Yz*M+I@VnI(WJ%y-K7AB$j!e-94)t-m`PMM+)X9rX_@O2VsU6+5I3g zu(xjsF+Q;UIMu_mS8T1%uZ$6`Ca|J`mEq(IZ0>==G|%eu-E;F*g8=T(CUdhPGQoENz{s^-L6)(?IM2$^9`D2d-aY>M9f!oo zfG=C}Q4Ek}tjxEc^QGQSi1lUS5lb~$D>wSDOK@| z1RgYu9sowxy~BFg>In)HI1vzyv6!VDB>~C98+SP0ZP=hHOfiBL%P9hhX`bxrm86`P zec&@G+^6Qldw(hr>(#}aW7tjuia-sh2)%0SjxYNDyPkR4XeDR9#r{!bm{9~0`(Ik) zRJG@jyAQ8l)oPYyk}B=knux`)fdRcZz`e%C5w;M>Gr8t?tP{A4zgcU}V7q+~Z8#fYReOe=rC>Dcg7xoyGD0a3v#e#k&0X8}(? zu~!F!e>gN)Fc@|ny>|6m%G3p9fZ2-`<|a@PaKC>Bk^VjobkYCuaAp@OD*mlwy z*;y|c$fxg}r=O`qi~4b#DxcoI8CmC zof0!bQFfW;IW5JurTPZF;|e`fobf|X?IzeSC^1|pogy*LvnE=a8C|e4;Y@CHs4Q)e zy44Ada4>tZ!X^bO!Z>;^Gs`8KbpDd>elIT41gEjHRx3=%+<(pJ~ zb_h!D+>Q+FGXW@kU~M7B=&j{<6Ak;hD7xB9Zg0`1Psj+S2qbQ9TEjCale0K(c7@uh z8usm_z5d83fyP2XS|hfMUvRVR`MhlllULNeumZioauxCs8VuMemN4>$UV|?}f1~D> zh(0;qZFilFjJMF3@7x~4<+WV-aKD_Jo4NeGI1e;yL$J>spfG_?t;85Nc+K|@%j;c_ zu8En_Pn>8*FDW3qMV`NCNST z%;ZBq{y26-9T}z+fy7x#Ydi}clpk7GYgH_>pO1BA=5=KBL4$Rklg#vcYlXoW6(x;r za{6+^Q(nkOga%s#ZH&g&KMU3^U-0*k}U8< z!Poo2u=i?cgkskQMW_x`1f!!mB6;rZHPP}s18%MN9AG9gkH)^exFDyL8>Tje5le4d!*U2KVNr^3M(( zZtdpzX{5i{dJ{ClA45ZsXC9;Ikvogk%~zWaUyFVe>j&-wHjZ%YoC^w59H@bDKWaS_P{6r6(L5f zz%KXL42kCI@(PtfYpz*jpgYLaO>4|nMV||F9IvTSWa#8f6#&mFCB_(N2?`5XBZ)Ca zCQ^!mK3$lz_6aG?;*JK_#*8!Y)W^`GNNbejtVnd*Ecj6<^}ZQNwvKYo!T)DKCo*R2 zP7=n^EZ6}m!pIBeykdKHJrG@fB&keCeE)7}?8(B~gTEF`f{I|2{9GaJE5OCU zvSE*Uf`0oIcn8zaZouJ9YskKRlJITu-VJU)i(4egxi-)^{{)Kw?gzH76JmrOc>VUW z-=fu#LvDe(<>GC~Xrc(DP1sSHU<8|gXSVivd15F+;+MCss|qwSyeJ04K@M8u>D-`G zI5|vc5qrzJ>}GMW$6`-n8`csWch6xtBgFU-^-k&NlzPe*t|OaczTFV~m%(t}gVxBk z>g#GVTg%UVG4og6x`QB_lo)6?;K-mgo~KO6OWD8Y`c;s{!XT3S1sM&XB`CZOpd$2Y z$242yL~&u8iG2)Lz}@1l(0GG22e*7~8q@?M`|U@yj+LXaLs@27M?b#?&n(ufHzE{+ zRAoeKXx-i1KD$-mY-jtSu)Yt=P_Ev9od+wzTR=sK@%pLkv8_uly=tAk{JuqUYZi*3 ziM0m?wo4O?d}B$8qq}m}3^MxidSrUzF?1*bNqC;~?s9IG5B)XLaXf_`6B%r6;3P>I z2ev|kBD@4tgcxe<@05?SWqZv|GH+PDtc?vB1{8rLjF|*ue_5SKvyt((KTG&rIyR+0 zLdFJU%(F)o&eWCdjtpo0JbFu{!rJRMGE|Y_P8&l?zx4iI-Qj0eyr*7=ibR(q0~;|Y z!pfi`^eQ&3&%IJ3{QlZ6I!;^7YQvGCg|!Dp8mI_{(}5+cVlNpNhgIzIDEmYP*QVqO z>pv)qzk`Zk_{}ivVGZjt(Qb*`u*+aK85!mjfyBu~YfM#y#~l9orS?;GFORPANDwm2 zpurYEYp{!4Tu_{}-zksrA-}w>44B`P`H=t(9&BwS^vdK)lll6WTTgsWa~$^Ed=2eU zZ!Dp)3Y%F3L#fV#eB_aG(CuTPGeO4fU8q;sXB$wM2S7zIjvwMK4AHLh$%!uw4~%dH QXK6~V@+bn!q7cmZA6^u=x&QzG literal 0 HcmV?d00001 diff --git a/.gradle/9.7.1/fileHashes/fileHashes.lock b/.gradle/9.7.1/fileHashes/fileHashes.lock new file mode 100644 index 0000000000000000000000000000000000000000..f20a654d84bb02bd450e64d42a4e3c94ecb4952c GIT binary patch literal 17 UcmZRM{Nbo5;HS;P00xXg03VkER{#J2 literal 0 HcmV?d00001 diff --git a/.gradle/9.7.1/fileHashes/resourceHashesCache.bin b/.gradle/9.7.1/fileHashes/resourceHashesCache.bin new file mode 100644 index 0000000000000000000000000000000000000000..971f431a2280905337bb75ec79d52e75032fc010 GIT binary patch literal 22577 zcmeI(XHXMuzXou6??tK<>AfmQ6CR|uK!DIYDp&xOj(~KODk8lJf+!-W6a@>QA}B?Q z0Yr+5ASws~DyY0>HaG8_Y-XOB^Wl6rvzb9NKQI5gdnKEV6ZqfogDebuP z$9%GN$k8jnQ_M%g->-~yKyJ?pJY6e{Z(LQ_A974B@Ql_Ljlv_Do{*ba0ne0RbSgoa zzJ=U|6L^-&3+!?nDhhJrPrxrMNuN=8=q?Vq2_x{F@>edA^u|MYKLz|^`5yn$yT@?H z{`X-U4m@`;(zjmyRRQD{qrmeWrDjI8YCq%sJMc0ZH=~y~RLdZ@yAHf+KQn{=AH!CB zJ{$PenVZob?8-0kE(5%FKUW8%v=DxdNiOhfipt!AuZ3PidE5ItK6XTgtD18+e!GW( z*ON;Qy-c(7fZWt`$Kx_PGY)XHL2j%EyrHp%epu${VaUzrf!_+|o4oh={$0q;egeOv zSux^xAchw2LBLz1D{YK)v1rKcQh~SEy}MxXv`z0;w1o(&rxyp&^ z`*V}S>U6-X6y3QBQGE~83&G)+L*&|(^=#5Pj>tWhkwT0xG&^Z zX28d#r;Jvw2H;14=ppFoPgZY z6*xtsic{=pZ#m=`U*J@|CstxhE$1P(Iu4w!*ouGhVM!L`_A0;`*=f1v0uoL_ZrTl; z^UBIrmR8jReEtn^o||Hnmt4c}?+0^j;QV16QE@8y!%*IKFYw(1le3}~WrFzm2Y?G4 z65@`J?Ysav<~eXd>GW&Xy=I3Xw~_)b!lUMK=w(p^iMdiJJ%%PgHA-A^!E>`QK zrNOZ@iJ#L5Tmp0LBv)U|6Ufm6z@^(C3(BE4gzBF%9JPPPi{|HzLLcIf*UkX= z{ucEtB_ke2C~y4|xYi-I*MTFs3HUi&z_q&yHg!KI{QW+Gdzfqi*Y#wq^AC#vuX{Xj zz3=@QM++tKpL?bsfgfabrqVpTd;*%|z`Wzl19#dA$7mq8aRhF-m7U_Ou6zt~+iBoN z1zlWBs165wz67{&iJR8vJXSu)?MQ)}e&N5jqD9{Vxj7$jt4gEJL196A$So{({LXxx z##q26IBx4CJVHJO0*y`?pNiJIFD@z#Y7QPrb0I zpN8D754fY{ar5;&8~pXfJ`}i{l*C~gOmH`pH^Tz=h|uJmTIIR{x#irBw~B4+k^I2F zE~ql#-m0bLNqhSZ?V!-`tHgtb!`iDVz^m*XN^4JwCS)_hLZb1wD zxZs2dzabeoUQOTuS0;nR^xxupv&8^E9r&7oqVRnUG{+_qc!cZM3kL&f8z9Gg0Upim z(_H9eTh^E`Ait6B!*eeLx5AVlksbb( zHgj^txB0=Y$YO(B+}dD*#4FwLoJ)I*8z@CEu1U_E%kTv1v(}q zA$u0cIx`>J@7>6?@b9mszmLMz?ys&ZVTL6G=lxmK#!H>@@Jdx{ND4sv&Isf?1 zV!#;`w>LaNwUMZKhPHT$IpKz2M;(O{Jkcyw#`l!XL}(;kc7`mHbq=0jc1&J89DlOm zt0)t91+C&4c%s~C$hBhCK`xf1j&@T{_zzzPE`Z%Y6l&I53*} z=xJ=SA8&lug9nLEiM)gOOQy-$%hIEF7mlt4-`9=KhcUcek?z4_l-ex zYi&L};nfkdw6ySw-@854XV66vcS{bzeq8p5NITx6)z`L5Zt)zuX9YYV6M#Kr_Kl@C zVt8_7tJlE@o~RLw_IP=FoYPQ;EO%fC9S={Wno~Q>{*3JWZ+v@dmZh|If@Mx;EAHN&~81$s%&z z3cZM`h+dfB$h#y&DyA*B0iWR_VSxRIi?`Ijg86&W!`yIqBK)CyMd|p`{rmcArzw}? zV&I8rk=rlU_=LEys@aa=-=aq0iKk(7t)wJ+%8873lvQ$W)xr~(*SLzk)BP0Kz4&?O zsvhFj@Dsc)l&cJ#7w;5j*c-9E^+_Hog(uSH>r_Rv4ZJdWmv#xB+`!$RK`_J2yGO13 zv$Awe!bHT0GcCB4#srD=rDRb|ihRe+>sd=BC)ovfqS^l0j8FWjHF74o+8)P7I0h zqTn+`+)cPFa^8qbX4?m!m2ttXNhWAaZ=X)C8x_{7(AP2lL&fqqJRz_zs4iS;N<*DP z(|()gF_E+3@eehHA3uNX#ZXb{9Z?8B4xgc|kW>6(W!QYXJC&Ek%Xtx=$TQ{p$>_(@ z@s3q{*qLV_2cEDS+;+Gas7xzzPVs8Ik53vr!B|n7_r29r*ffLOT2xpRw<4e5XalIT z+ejW=t;r5e957dF`3z48txNXTDs%+D)+t=p@6{TFCl2k)DLYtp*~vgsS3-S^l*ko% zu|XrV#O*53>!Q_qvyLxQ~Rx(Eq*Y^o-`&Fp5SiI6p%aD-+IG+ z>6x$ONmqCxVqxZzrM$Qrhw=08r>Q6k;ECtc=B8POiLy~MCIZ%}9gpA%aYk!dtz&1z zZ0TzlW3OFmn<{b zqB2Qznc!S%&VGJvjoCbSqLEs^R%DvZE3BI0nVa-(BA+mQJlsKQUtf2r%4FM_T9TsS zGkys$DA!z@)<`FJmssj+-w#hvDV-=uTPPMciq%xJzMF=DCwR9|dlkLeUa4+bc^J*n z6Zu>p+l@N^%jeaBf=^X(ZFj2W;4_52SWo1d_!!kll(jaAx4(cVQZ6jgR=f$ADZR81 zZW5vx1W&NJ%)P!VLrNp^i{76}=RA>5{wZ`}@pF#^E;0t0b}^wM-2Gt$XQlCCiko7b zO4^|kX}z}QLt5~}o({~m*!T;YN5)BUfwzjX;ECIOTayY?C&T|CJ740Wx$Ff`&>s#< zcg%*CA!oy&zsm+Ru_xORm=;o!xM8cF0D0B>O>V{xSCb^n1kVodP_Rt25GU0c66+9 z)n*^=4lshPU_`o)4$0l956*9>XE;_*2C{QZ+Yz_()2ug^HN${-%>{2Dd zs}MyCf6*qUH+gy&dk}yC1Rwwb2tWV=5P$##AOHafKmY;|fB*y_009X6F9NRAg=m$F zDJqrHJB0xu%JWk@hLGX#UY)5YS2^r7@$UzrUvk129a%?xHFwF- z=8&Hby{Dewu1UP>h>RwxsXyVKSEp-{j=i9H=ML`qlZ{=68t&bu`F8G0xA~_Q54TZw zR&rmqv;RwS(Ir{fH|3y7gi#hvtj9*AJ`fdu#h!Xx_v9RJ2{M)99Db z{C)2Jv5I-Yi2pA2)7<%02*s$ywPmnerRV7Mwc( literal 0 HcmV?d00001 diff --git a/.gradle/vcs-1/gc.properties b/.gradle/vcs-1/gc.properties new file mode 100644 index 00000000..e69de29b diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..4729dd8b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# Stage 1: Build +FROM eclipse-temurin:23-jdk AS build + +WORKDIR /app + +COPY gradlew . +COPY gradle gradle +COPY build.gradle settings.gradle ./ + +COPY src src + +RUN --mount=type=cache,target=/root/.gradle \ + chmod +x gradlew && ./gradlew bootJar --no-daemon + + +# Stage 2: Runtime +FROM eclipse-temurin:23-jre + +WORKDIR /app + +COPY --from=build /app/build/libs/*.jar app.jar + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --system --create-home messenger + +USER messenger + +EXPOSE 8080 + +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ + CMD curl --fail --silent --show-error http://localhost:8080/health/ready || exit 1 + +ENTRYPOINT ["java", "-jar", "app.jar"] diff --git a/Makefile b/Makefile index 5d86d171..c1d00b88 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test test-idempotency load test-persistence test-failover test-failover-each test-rolling swarm-up test-node-failover swarm-down help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -18,13 +18,29 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-idempotency: + $(PYTHON) student_tests/idempotency_concurrent.py --base-url "$(BASE_URL)" $(TLS_ARGS) + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) test-persistence: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-failover-each: + $(PYTHON) student_tests/failover_each.py --action-script "$(ACTION_SCRIPT)" --restore-script "$(RESTORE_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-rolling: + $(PYTHON) student_tests/rolling_update.py --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +swarm-up: + powershell -NoProfile -ExecutionPolicy Bypass -File scripts/swarm-lab-up.ps1 + +test-node-failover: + $(PYTHON) student_tests/node_failover.py --base-url "http://localhost:18080" --output "evidence/node-failover.json" + +swarm-down: + powershell -NoProfile -ExecutionPolicy Bypass -File scripts/swarm-lab-down.ps1 diff --git a/README.md b/README.md index 10381de4..1ec655ca 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,119 @@ **Архитектура:** Клиент → LB → API × 2+ → общие PostgreSQL и хранилище сессий. Compose или Swarm — на выбор. +## Реализованный запуск Compose + Nginx + +В этой реализации публичный порт есть только у Nginx. API масштабируется как +один Compose-service без `container_name`; сессии, сообщения и результаты +идемпотентности находятся в общей PostgreSQL. Redis используется только как +необязательный общий cache с безопасным fallback в PostgreSQL. + +Из чистого clone выполните одну последовательность (после копирования +`.env.example` замените локальный пароль): + +```sh +cp .env.example .env +# отредактировать DATABASE_PASSWORD в .env +docker compose --env-file .env build api +docker compose --env-file .env up -d postgres redis +docker compose --env-file .env run --rm migrate +docker compose --env-file .env up -d --scale api=2 api nginx +curl --fail http://localhost:8080/health/ready +``` + +`migrate` — отдельный one-shot процесс; в API Flyway отключён. При обновлении +схемы сначала выполняется совместимая миграция той же командой, затем API +заменяются по одной. Nginx 1.27.4 обновляет адреса `api` через Docker DNS каждые +2 секунды, применяет `least_conn`, не использует sticky session и повторяет +ошибку соединения/502/503/504 на другой реплике. Readiness проверяет завершение +V3-схемы, доступность PostgreSQL и состояние Spring `ACCEPTING_TRAFFIC`. + +Проверки после запуска: + +```sh +make test +make test-idempotency +chmod +x scripts/stop-one.sh scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +``` + +В Windows PowerShell те же проверки можно выполнить с +`scripts/stop-one.cmd` и `scripts/restart.cmd`; `.sh` остаются основными +POSIX hooks для CI/защиты. + +`test-idempotency` одновременно отправляет 32 одинаковых запроса и требует, +чтобы ответы пришли минимум от двух `X-Instance-Id`, но в истории появилось +ровно одно сообщение. `stop-one.sh` сопоставляет полученный `X-Instance-Id` с +hostname контейнера и останавливает только эту API-реплику. После демонстрации +её можно вернуть командой: + +```sh +docker compose --env-file .env up -d --scale api=2 api +``` + +### Проверки уровня 4 + +Обе реплики последовательно останавливаются с восстановлением между опытами: + +```sh +make test-failover-each \ + ACTION_SCRIPT=scripts/stop-one.sh \ + RESTORE_SCRIPT=scripts/restore-api.sh +``` + +В Windows используйте `.cmd` варианты. Тест сохраняет по каждой реплике +successful RPS, число ошибок и время первой успешной операции после stop. + +Эквивалент rolling update для Compose — последовательная замена каждого +исходного API-контейнера с ожиданием readiness новой реплики перед заменой +следующей. Проверка одновременно держит четыре потока записи: + +```sh +make test-rolling ACTION_SCRIPT=scripts/rolling-update.sh +``` + +Nginx обновляет Docker DNS раз в 2 секунды. Сетевой отказ определяется не +позднее connect timeout 500 ms, зависший ответ — read timeout 4 s; адрес +пассивно исключается на 2 s после первой ошибки. Поэтому верхняя граница +обнаружения одного неотвечающего backend — 4 s, а две upstream-попытки +укладываются примерно в 8.5 s. Docker readiness каждой API проверяется раз в +2 s с timeout 2 s. Nginx OSS использует readiness при начальном/операторском +запуске через Compose и пассивное исключение во время работы. + +Для `POST /chats/{id}/messages` Nginx может повторить запрос на второй backend +даже после начала отправки: тело буферизуется, а обязательный общий +`Idempotency-Key` делает такой retry безопасным. Для остальных POST параметр +`non_idempotent` не включён, чтобы не размножать пользователей или чаты. + +## Отказ узла: отдельный трёхузловой Swarm-стенд + +Дополнительные студенческие проверки находятся в `student_tests/`; исходные +тесты курса в `tests/` не перемещаются, Spring-тесты остаются в `src/test/java`. + +Для трека отказа узла добавлен одноразовый локальный стенд из трёх настоящих +Docker daemons в privileged DinD-контейнерах: manager и два workers. Он не +заменяет основной Compose-стенд и использует отдельный порт `18080`. + +```powershell +make swarm-up +make test-node-failover PYTHON=python +make swarm-down +``` + +`swarm-up` строит образы из текущего исходного кода, публикует их в одноразовый +локальный registry, доступный всем узлам, +создаёт overlay network, отдельно выполняет миграцию и запускает API с двумя +replicas и `max_replicas_per_node=1`. Тест выбирает worker с API-задачей и +жёстко останавливает весь Docker daemon этого узла. Swarm должен создать новую +API-задачу на свободном узле; через LB одновременно выполняются чтения/записи, +после чего проверяются старые и подтверждённые во время отказа сообщения. + +Защищён только API worker. PostgreSQL, Redis и Nginx закреплены на manager; +PostgreSQL использует локальный volume и не реплицирован. Потеря manager/DB +узла не переживается и в отчёте не заявляется. Один manager также означает +SPOF control plane; DinD underlay и физический host остаются SPOF эксперимента. + ## Что сделать 1. Выберите Compose + Nginx/HAProxy/Traefik либо Docker Swarm. Для выбранного варианта дайте одну последовательность запуска. diff --git a/REPORT.md b/REPORT.md index c2679005..d1f696a4 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,429 @@ -# Отчёт — лабораторная 3 +# Отчёт по лабораторной работе №3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Тема:** горизонтальное масштабирование, балансировка и отказоустойчивость API -## Запуск из чистого clone +**Автор:** Антипин Егор Иванович, БПИ2301 -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Целевая оценка:** 5 -## Архитектура и решения +**Выбранный трек уровня 5:** трек 2 — «Отказ узла» -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +**Commit/PR:** работа проверена в локальной копии; ссылку следует указать после публикации репозитория. -## Доказательства критериев +## 1. Цель работы -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Цель — запустить несколько взаимозаменяемых экземпляров API за одной публичной +точкой входа и сохранить корректность приложения при остановке отдельной +реплики. Сессии, сообщения и результаты идемпотентных операций не должны +зависеть от памяти или локального диска API. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Для уровня 5 дополнительно проверяется потеря целого Docker-узла. Реализованы +два стенда: -## Эксперименты и отказы +1. основной Compose-стенд с Nginx и двумя API-репликами; +2. отдельный трёхузловой Swarm-стенд для переноса API после потери worker-узла. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## 2. Использованные технологии -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +- Java 23, Spring Boot 4.1.1; +- PostgreSQL 16.4 — источник истины; +- Redis 7.4 — общий необязательный cache-aside кэш; +- Nginx 1.27.4 — единственная публичная точка входа и балансировщик; +- Docker Compose v2 — основной стенд; +- Docker Swarm и Docker-in-Docker — эксперимент с отказом узла; +- Flyway — версионирование схемы; +- Python 3.9+ — black-box и нагрузочные сценарии. -## Ограничения и следующие шаги +Приложение собирается из исходного кода текущего репозитория. Готовый образ +серверной части преподавателя не используется. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## 3. Архитектура основного стенда + +```text +клиент -> localhost:8080 -> Nginx -> API replica 1/2 -> PostgreSQL + | + +-----------> Redis (необязательный кэш) +``` + +Nginx — единственный сервис, публикующий порт на host. API, PostgreSQL и Redis +доступны только во внутренней Docker-сети `backend`. У сервиса `api` нет +`container_name`, поэтому один Compose-service масштабируется параметром +`--scale api=2`. + +### 3.1. Роли компонентов + +| Роль | Service / образ | Health/readiness | Хранилище | Порт и сеть | Credentials | +| --- | --- | --- | --- | --- | --- | +| Публичный LB | `nginx`, `nginx:1.27.4-bookworm` | `nginx -t`, каждые 2 s | нет | `127.0.0.1:8080 -> 80`, `backend` | не требуются | +| API | `api`, локальный `hla-lab3-api:local` | `GET /health/ready`, каждые 2 s | нет | только `8080/backend` | параметры БД из `.env` через environment | +| Миграции | `migrate`, тот же образ API | one-shot, код завершения | нет | `backend` | те же параметры БД | +| Источник истины | `postgres`, `postgres:16.4-bookworm` | `pg_isready`, каждые 2 s | `postgres-data` | только `5432/backend` | `DATABASE_PASSWORD` из `.env` | +| Общий кэш | `redis`, `redis:7.4-bookworm` | `redis-cli ping`, каждые 2 s | `redis-data`, AOF everysec | только `6379/backend` | наружу порт не открыт | + +PostgreSQL хранит пользователей, Bearer-сессии, чаты, членство, сообщения и +`message_idempotency`. Поэтому следующий запрос одного пользователя может +попасть на другую API-реплику без повторного входа. Redis не является +источником корректности: при cache miss или недоступности Redis приложение +читает данные из PostgreSQL. + +Постоянные volumes подключены только к PostgreSQL и Redis. API и Nginx +stateless и могут быть удалены и созданы заново. + +## 4. Воспроизводимый запуск Compose + Nginx + +Требуются Docker Engine с Compose v2, `make` и Python 3.9+. + +```sh +cp .env.example .env +# заменить DATABASE_PASSWORD в .env локальным случайным значением +docker compose --env-file .env config --quiet +docker compose --env-file .env build api +docker compose --env-file .env up -d postgres redis +docker compose --env-file .env run --rm migrate +docker compose --env-file .env up -d --scale api=2 api nginx +curl --fail http://localhost:8080/health/ready +``` + +Это единая последовательность запуска. Миграции выполняются отдельным +one-shot сервисом до API. Во всех обычных API-репликах установлено +`FLYWAY_ENABLED=false`, поэтому реплики не соревнуются за схему. Повторный +запуск `migrate` безопасен: Flyway сверяет таблицу истории и не повторяет уже +применённые версии. + +Обычная остановка сохраняет данные: + +```sh +docker compose --env-file .env stop +``` + +Удаление учебных данных выполняется только осознанно: + +```sh +docker compose --env-file .env down --volumes +``` + +`.env` не должен попадать в Git. Для production вместо него требуется secret +manager или Docker secrets. + +## 5. Балансировка Nginx + +Конфигурация находится в `infra/nginx.conf`. Upstream задаётся DNS-именем +`api:8080`. Встроенный Docker DNS `127.0.0.11` возвращает адреса актуальных +Compose-реплик. `resolve` и `resolver ... valid=2s` обновляют список адресов +без reload с TTL 2 секунды. + +Алгоритм `least_conn` выбирает живую реплику с наименьшим числом активных +соединений. Реплика с длинными запросами временно получает меньше новой работы. + +Sticky session не используется. Она закрепляла бы клиента за одной репликой и +скрывала ошибочное локальное состояние. Здесь запросы одного Bearer-пользователя +могут проходить через разные реплики. `X-Instance-Id`, равный hostname +процесса, позволяет это наблюдать. + +Параметры upstream: + +- `max_fails=1`: одной сетевой ошибки достаточно для временного исключения; +- `fail_timeout=2s`: ошибочный peer исключается на 2 секунды; +- `proxy_connect_timeout=500ms`; +- `proxy_read_timeout=4s` и `proxy_send_timeout=4s`; +- максимум две upstream-попытки. + +Для зависшего backend максимальное время первичного обнаружения — 4 секунды. +Две последовательные timeout-попытки завершают запрос примерно за 8,5 секунды. +Ошибка соединения определяется быстрее — в пределах 500 ms на попытку. Это +укладывается в требование возобновить новые операции не позднее 10 секунд. + +Nginx OSS использует пассивную runtime-проверку upstream: Docker readiness не +допускает ранний операторский запуск публичного маршрута, а отказ уже +работающего backend обнаруживается запросом и указанными таймерами. + +## 6. Readiness и graceful shutdown + +`GET /health/live` подтверждает, что процесс работает. `GET /health/ready` +проверяет: + +1. Spring `ReadinessState.ACCEPTING_TRAFFIC`; +2. доступ к PostgreSQL; +3. наличие таблицы `message_idempotency`, то есть применение V3. + +Ping Redis прогревает соединение, но ошибка Redis не делает API неготовым, +поскольку Redis — необязательный кэш. + +Docker вызывает readiness каждые 2 s, timeout — 2 s, start period — 45 s. +Перед завершением Spring переводит приложение из `ACCEPTING_TRAFFIC`, +прекращает приём новой работы и завершает принятую. В `application.yml` +установлены `server.shutdown=graceful` и +`timeout-per-shutdown-phase=10s`; Compose даёт API 12 s через +`stop_grace_period`. У Nginx `worker_shutdown_timeout=10s` и такой же +`stop_grace_period`. + +## 7. Общая идемпотентность сообщений + +Для `POST /api/v1/chats/{chatId}/messages` обязателен `Idempotency-Key`. +Область ключа: + +```text +(user_id, HTTP method, canonical request path, Idempotency-Key) +``` + +В общей PostgreSQL-транзакции API вычисляет SHA-256 семантического тела и +получает transaction-scoped advisory lock: + +```sql +SELECT pg_advisory_xact_lock(hashtextextended(scope, 0)); +``` + +Advisory-блокировка — именованная блокировка PostgreSQL, которой управляет +приложение. `pg_advisory_xact_lock` действует до конца транзакции. Одинаковый +ключ с разных API-реплик сериализуется общей БД. Затем сервис: + +1. ищет запись в `message_idempotency`; +2. при совпадающем SHA-256 возвращает ранее созданное сообщение; +3. при другом теле отвечает `409 Conflict`; +4. при новом ключе создаёт `messages` и `message_idempotency` в одной + транзакции. + +Сообщение и запись идемпотентности фиксируются атомарно. Если клиент не получил +ответ, он безопасно повторяет запрос с тем же ключом через любую реплику. +Записи живут не менее 24 часов. Коллизия 64-битного хеша advisory lock только +лишний раз сериализует разные ключи: полная область проверяется primary key. + +Nginx разрешает `non_idempotent` retry только для endpoint сообщения. Тело +буферизуется, а общая идемпотентность делает повтор безопасным. Остальные POST +после начала отправки автоматически не повторяются. + +Результат `student_tests/idempotency_concurrent.py`: 32 одновременных запроса с +одним ключом прошли через два `X-Instance-Id` и создали ровно одно сообщение. + +## 8. Проверка отказа API-реплик + +`scripts/stop-one.sh` (Windows: `stop-one.cmd`) получает наблюдаемый +`X-Instance-Id`, сопоставляет его hostname контейнера и останавливает выбранную +API-реплику. Нагрузка идёт только через Nginx. + +```sh +chmod +x scripts/stop-one.sh scripts/restore-api.sh +make test-failover-each \ + ACTION_SCRIPT=scripts/stop-one.sh \ + RESTORE_SCRIPT=scripts/restore-api.sh +``` + +Предыдущая реплика восстанавливается и снова наблюдается через LB до остановки +следующей. В Windows используются одноимённые `.cmd`-скрипты. + +Фактические результаты 03.11.2026: + +| Реплика | Чтения | Записи | Ошибки | Successful RPS | Первая успешная операция после stop | +| --- | ---: | ---: | ---: | ---: | ---: | +| `170796a1dad0` | 81 | 82 | 0 | 10,839 | 1,306 s | +| `feb12e05fd43` | 75 | 76 | 0 | 10,014 | 2,672 s | + +Обе реплики до опытов были видны по `X-Instance-Id`. Во время отказа +продолжались чтения и идемпотентные записи, подтверждённые сообщения остались +доступны. Raw-результат: `evidence/failover-each-2026-11-03.json`. + +## 9. Persistence и сохранение сессии + +```sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +``` + +`restart.sh` перезапускает зависимости без удаления volumes, выполняет +однократную миграцию и поднимает две API-реплики. Тест использует созданный до +рестарта Bearer token и проверяет прежние данные. Корректность обеспечивает +PostgreSQL, а не память API и не Redis. + +## 10. Ресурсы и изоляция + +- каждая Compose API-реплика: 0,75 CPU и 768 MiB; +- Nginx: 0,25 CPU и 128 MiB; +- API не публикует host port; +- volumes есть только у PostgreSQL и Redis; +- `container_name` отсутствует; +- API/Nginx не имеют состояния, необходимого для корректности. + +В Swarm PostgreSQL ограничен 1 CPU/768 MiB, Redis — 0,25 CPU/256 MiB. + +## 11. Rolling update под нагрузкой + +`scripts/rolling-update.sh` по очереди заменяет каждую исходную API-реплику и +ждёт readiness новой. Одновременно `student_tests/rolling_update.py` держит +четыре закрытых потока записи: + +```sh +make test-rolling ACTION_SCRIPT=scripts/rolling-update.sh +``` + +Фактический опыт 03.11.2026: + +- нагрузка — 180,118 s, замена — 115,800 s; +- наблюдались четыре `X-Instance-Id`: две старые и две новые реплики; +- 12 147 успешных записей и 177 ошибок; +- successful RPS — 67,439; error ratio — 1,4362%; +- latency успешных запросов: p50 37,729 ms, p95 189,374 ms, + p99 215,986 ms. + +Ошибки не скрыты клиентским retry. Это closed-loop измерение на одном host; +percentiles не включают ошибки и подвержены coordinated omission. Raw: +`evidence/rolling-update-2026-11-03.json`. + +V3 обратно совместима: она только добавляет таблицу и индекс. Старая версия +работает после применения V3, новая не становится ready до появления V3. +Автоматический downgrade не реализован; rollback приложения оставит +неиспользуемую V3-таблицу. + +## 12. Уровень 5: отказ целого узла + +### 12.1. Гипотеза + +Если две stateless API-задачи распределены по разным Swarm-узлам, есть +свободный узел, а общий PostgreSQL работает, потеря worker с API приведёт к +созданию новой задачи на живом узле. Через Nginx продолжатся чтение и запись, а +подтверждённые данные сохранятся. + +Гипотеза не утверждает выживание узла PostgreSQL: БД не реплицирована. + +### 12.2. Топология и образы + +Одноразовый стенд содержит три независимых Docker daemon в privileged DinD: +`hla-swarm-manager`, `hla-swarm-worker1`, `hla-swarm-worker2`. + +API имеет две задачи, `endpoint_mode: dnsrr` и +`max_replicas_per_node: 1`. Третий узел оставляет место для переноса. Nginx +обращается к `tasks.hla-app_api:8080` через overlay DNS. + +`scripts/swarm-lab-up.ps1` собирает API и Nginx из текущего репозитория, +публикует `hla-swarm-registry:5000/hla-lab3-*:swarm` в одноразовый registry и +разворачивает образы на узлах по digest. Затем создаётся overlay network, +разворачивается state stack, отдельно выполняется migration job и запускается +app stack. + +```powershell +make swarm-up +make test-node-failover PYTHON=python +make swarm-down +``` + +Swarm Nginx доступен на `http://localhost:18080`, отдельно от Compose на 8080. + +### 12.3. Методика + +`student_tests/node_failover.py`: + +1. проверяет не менее двух API tasks и instance ID; +2. создаёт контрольное сообщение; +3. находит worker, реально выполняющий API task; +4. жёстко останавливает весь внешний DinD-контейнер worker; +5. 30 s выполняет через Nginx чтения и идемпотентные записи; +6. проверяет новую task ID на другом узле и новый `X-Instance-Id`; +7. читает контрольное и все подтверждённые сообщения; +8. возвращает worker и ждёт `Ready`. + +### 12.4. Результат + +03.11.2026 полностью остановлен `hla-swarm-worker2`. + +| Параметр | Результат | +| --- | --- | +| Старая task | `sesmka0dxn5k` на `hla-swarm-worker2` | +| Новая task | `pke5uf5okm69` на `hla-swarm-worker1` | +| Instance ID до | `1726221ff80d`, `edb5532a721e` | +| Instance ID после | `1726221ff80d`, `81c7197f2bfe` | +| Успешные чтения / записи | 165 / 166 | +| Ошибки | 0 | +| Первая успешная операция | 1,796 s | +| Successful RPS | 11,033 | +| Полное время с возвратом узла | 73,763 s | +| Подтверждённые данные сохранены | да | + +API-задача реально перенесена между Docker daemon. Контрольное сообщение и все +подтверждённые во время отказа операции сохранились. Результаты: +`evidence/node-failover-2026-11-03.json` и +`evidence/node-failover-2026-11-03.md`. + +В JSON записано `database_node_failed=false`: узел БД не падал, и выживание +PostgreSQL не заявляется. + +### 12.5. Оставшиеся SPOF + +| SPOF | Причина | Что требуется для production | +| --- | --- | --- | +| Nginx | одна задача на manager | несколько LB за внешним L4/VIP или managed LB | +| PostgreSQL | одна реплика на manager | репликация, автоматический failover, проверка RPO/RTO | +| Redis | одна реплика | Sentinel/Cluster, если требуется HA кэша | +| Swarm control plane | один manager | 3 или 5 manager-узлов для Raft quorum | +| Storage | локальные volumes manager | реплицированное/сетевое хранилище и fencing | +| Underlay/host | DinD на одном компьютере | разные hosts и резервирование сети | + +Доказан failover API-worker при живом state/LB-узле. Потеря manager, DB, LB, +storage или физического host не переживается и не заявляется. + +## 13. Команды проверок и доказательства + +Дополнительные проверки находятся в `student_tests`; исходные тесты курса — в +`tests`, Spring-тесты — в `src/test/java`. + +| Требование | Команда / доказательство | Результат | +| --- | --- | --- | +| Структура и синтаксис | `make check` | PASS | +| Основной API через LB | `make test` | PASS | +| Общая идемпотентность | `make test-idempotency` | 32 запроса, 2 реплики, 1 сообщение | +| Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | прежние token и данные доступны | +| Выбранная реплика | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | чтение/запись продолжаются | +| Каждая реплика | `make test-failover-each ...` | 0 ошибок, recovery 1,306/2,672 s | +| Rolling update | `make test-rolling ...` | 4 instance ID, 12 147 успешных записей | +| Потеря worker | `make test-node-failover` | task перенесена, 0 ошибок, данные сохранены | +| Compose config | `docker compose --env-file .env config --quiet` | exit 0 | + +Сводка уровня 4: `evidence/level4-2026-11-03.md`. Числа опытов сохранены в +машиночитаемых JSON в `evidence/`. + +На Windows Python может использовать cp1251. Для чтения UTF-8 OpenAPI: + +```powershell +$env:PYTHONUTF8='1' +make test PYTHON=python +``` + +## 14. Соответствие критериям + +### Уровень 3 + +- `compose.yaml`, две API-реплики, одна публичная точка, общие БД и сессии; +- запросы одного пользователя идут на разные `X-Instance-Id` без нового login; +- отказ любой API-реплики проверен при чтении и записи; +- конкурентный повтор `Idempotency-Key` не создаёт дубль; +- smoke, persistence и failover выполняются через LB. + +### Уровень 4 + +- readiness, DNS refresh, timeouts и graceful shutdown согласованы; +- обе реплики остановлены по очереди с RPS, errors и recovery; +- заданы CPU/RAM limits, API-порты не опубликованы, `container_name` нет; +- rolling replacement проверен под нагрузкой; +- миграция запускается отдельно и является additive/backward-compatible. + +### Уровень 5, трек «Отказ узла» + +- создано три Docker-узла; +- потерян узел, реально выполнявший API task; +- Swarm создал новую task на другом daemon; +- чтение/запись продолжались, данные сохранились; +- разобраны SPOF LB, DB, control plane и storage; +- отказ DB не проверялся и не заявлен. + +## 15. Ограничения + +Измерения выполнены локально на Windows с Docker Desktop. DinD даёт отдельные +Docker daemon и реальный Swarm rescheduling, но не моделирует отключение разных +физических серверов. Результаты нагрузки зависят от host и не являются +production benchmark. Nginx OSS использует пассивное runtime-исключение. TLS и +внешний HA ingress в лабораторный стенд не входят. + +## 16. Использование ИИ и внешней помощи + +ИИ использовался для обсуждения Compose/Nginx, readiness/graceful shutdown, +общей идемпотентности, дополнительных сценариев `student_tests`, Swarm-опыта и +редактирования отчёта. Поведение подтверждено запускаемыми тестами и raw JSON. +Заимствованный прикладной код стороннего сервера не использовался; применены +стандартные open-source зависимости из build-файлов проекта. diff --git a/build.gradle b/build.gradle new file mode 100644 index 00000000..5910077b --- /dev/null +++ b/build.gradle @@ -0,0 +1,55 @@ +plugins { + id 'java' + id 'org.springframework.boot' version '4.1.1' + id 'io.spring.dependency-management' version '1.1.7' +} + +group = 'com.example' +version = '0.0.1-SNAPSHOT' + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(23) + } +} + +repositories { + mavenCentral() +} + +dependencies { +// implementation 'org.springframework.boot:spring-boot-starter-data-jpa' +// implementation 'org.springframework.boot:spring-boot-starter-security' + implementation 'org.springframework.boot:spring-boot-starter-webmvc' + implementation 'org.springframework.boot:spring-boot-starter-validation' + implementation 'org.springframework.boot:spring-boot-starter-jdbc' + implementation 'org.springframework.boot:spring-boot-starter-data-redis' + implementation 'org.springframework.boot:spring-boot-flyway' + implementation 'org.flywaydb:flyway-core' + implementation 'org.flywaydb:flyway-database-postgresql' + implementation 'org.springframework.boot:spring-boot-starter-security' + implementation 'org.bouncycastle:bcprov-jdk18on:1.84' + implementation 'org.apache.commons:commons-pool2' + runtimeOnly 'org.postgresql:postgresql' + + implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' + implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.8.3' + compileOnly 'org.projectlombok:lombok' + developmentOnly 'org.springframework.boot:spring-boot-devtools' + annotationProcessor 'org.projectlombok:lombok' +// testImplementation 'org.springframework.boot:spring-boot-starter-data-jpa-test' +// testImplementation 'org.springframework.boot:spring-boot-starter-security-test' + testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation 'org.springframework.boot:spring-boot-starter-webmvc-test' + testCompileOnly 'org.projectlombok:lombok' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' + testAnnotationProcessor 'org.projectlombok:lombok' + + implementation("com.github.f4b6a3:uuid-creator:6.1.1") + +} + +tasks.named('test') { + useJUnitPlatform() +} + diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..de14a7f5 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,122 @@ +name: hla-lab3 + +x-api-environment: &api-environment + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL: jdbc:postgresql://postgres:5432/${DATABASE_NAME:-messenger} + DATABASE_USER: ${DATABASE_USER:-messenger} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:?Set DATABASE_PASSWORD in .env} + DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-10} + DATABASE_MIN_IDLE: ${DATABASE_MIN_IDLE:-2} + DATABASE_QUERY_TIMEOUT_SECONDS: ${DATABASE_QUERY_TIMEOUT_SECONDS:-5} + DATABASE_TRANSACTION_TIMEOUT_SECONDS: ${DATABASE_TRANSACTION_TIMEOUT_SECONDS:-5} + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_CONNECT_TIMEOUT: ${REDIS_CONNECT_TIMEOUT:-1s} + REDIS_COMMAND_TIMEOUT: ${REDIS_COMMAND_TIMEOUT:-500ms} + REDIS_POOL_MAX_ACTIVE: ${REDIS_POOL_MAX_ACTIVE:-8} + REDIS_POOL_MAX_IDLE: ${REDIS_POOL_MAX_IDLE:-8} + REDIS_POOL_MIN_IDLE: ${REDIS_POOL_MIN_IDLE:-0} + REDIS_POOL_MAX_WAIT: ${REDIS_POOL_MAX_WAIT:-500ms} + USER_CACHE_TTL_SECONDS: ${USER_CACHE_TTL_SECONDS:-300} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + +services: + postgres: + image: postgres:16.4-bookworm + environment: + POSTGRES_DB: ${DATABASE_NAME:-messenger} + POSTGRES_USER: ${DATABASE_USER:-messenger} + POSTGRES_PASSWORD: ${DATABASE_PASSWORD:?Set DATABASE_PASSWORD in .env} + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + networks: [backend] + + redis: + image: redis:7.4-bookworm + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 30 + restart: unless-stopped + networks: [backend] + + # One-shot Flyway runner. It is invoked explicitly before API replicas. + migrate: + image: hla-lab3-api:local + environment: + <<: *api-environment + FLYWAY_ENABLED: "true" + MIGRATE_ONLY: "true" + SPRING_MAIN_WEB_APPLICATION_TYPE: none + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + restart: "no" + networks: [backend] + + api: + image: hla-lab3-api:local + build: + context: . + environment: + <<: *api-environment + FLYWAY_ENABLED: "false" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://localhost:8080/health/ready"] + interval: 2s + timeout: 2s + retries: 15 + start_period: 45s + init: true + stop_grace_period: 12s + restart: unless-stopped + cpus: 0.75 + mem_limit: 768m + networks: [backend] + + nginx: + image: nginx:1.27.4-bookworm + ports: + - "127.0.0.1:${APP_PORT:-8080}:80" + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "nginx", "-t"] + interval: 2s + timeout: 2s + retries: 15 + stop_grace_period: 10s + restart: unless-stopped + cpus: 0.25 + mem_limit: 128m + networks: [backend] + +networks: + backend: + driver: bridge + +volumes: + postgres-data: + redis-data: diff --git a/evidence/failover-each-2026-11-03.json b/evidence/failover-each-2026-11-03.json new file mode 100644 index 00000000..b248ffcc --- /dev/null +++ b/evidence/failover-each-2026-11-03.json @@ -0,0 +1,24 @@ +{ + "scenario": "sequential failover of each API replica", + "replicas": 2, + "results": [ + { + "target_instance_id": "170796a1dad0", + "window_seconds": 15.038, + "successful_reads": 81, + "successful_writes": 82, + "failed_attempts": 0, + "successful_rps": 10.839, + "first_recovery_seconds": 1.306 + }, + { + "target_instance_id": "feb12e05fd43", + "window_seconds": 15.079, + "successful_reads": 75, + "successful_writes": 76, + "failed_attempts": 0, + "successful_rps": 10.014, + "first_recovery_seconds": 2.672 + } + ] +} diff --git a/evidence/level4-2026-11-03.md b/evidence/level4-2026-11-03.md new file mode 100644 index 00000000..797e61aa --- /dev/null +++ b/evidence/level4-2026-11-03.md @@ -0,0 +1,49 @@ +# Lab3 level 4 evidence — 2026-11-03 + +Environment: Windows host, Docker Desktop Engine 28.0.1, Compose deployment on +one machine, two API replicas limited to 0.75 CPU / 768 MiB each. All requests +used Nginx at `http://localhost:8080`; API ports were not published. + +## Sequential failover + +Command: + +```text +python student_tests/failover_each.py --action-script scripts/stop-one.cmd \ + --restore-script scripts/restore-api.cmd --window 15 \ + --output evidence/failover-each-2026-11-03.json +``` + +| Target instance | Successful reads | Successful writes | Errors | Successful RPS | First recovery | +| --- | ---: | ---: | ---: | ---: | ---: | +| `170796a1dad0` | 81 | 82 | 0 | 10.839 | 1.306 s | +| `feb12e05fd43` | 75 | 76 | 0 | 10.014 | 2.672 s | + +The stopped replica was restored and observed again before the next target was +stopped. Confirmed messages remained readable after both failures. + +## Shared idempotency + +`python student_tests/idempotency_concurrent.py` passed: 32 concurrent requests with +one key used two API instance IDs and created exactly one message. + +## Sequential replacement under load + +Command: + +```text +python student_tests/rolling_update.py --action-script scripts/rolling-update.cmd \ + --duration 180 --concurrency 4 \ + --output evidence/rolling-update-2026-11-03.json +``` + +- Rolling action: 115.800 s. +- Four instance IDs observed: two original and two replacements. +- Successful writes: 12,147; failed attempts: 177. +- Successful RPS: 67.439; error ratio: 1.4362%. +- Successful latency: p50 37.729 ms, p95 189.374 ms, p99 215.986 ms. + +This is a closed-loop single-host experiment. It includes coordinated omission +and client overhead; latency percentiles cover successes only. Brief failures +are retained in the result rather than hidden by client retries. PostgreSQL, +Nginx and the Docker host remain single points of failure. diff --git a/evidence/node-failover-2026-11-03.json b/evidence/node-failover-2026-11-03.json new file mode 100644 index 00000000..beab7e20 --- /dev/null +++ b/evidence/node-failover-2026-11-03.json @@ -0,0 +1,42 @@ +{ + "scenario": "hard loss of one Swarm worker", + "failed_node": "hla-swarm-worker2", + "old_task_id": "sesmka0dxn5k", + "tasks_before": [ + { + "task_id": "zmboecjxvdyn", + "node": "hla-swarm-manager" + }, + { + "task_id": "sesmka0dxn5k", + "node": "hla-swarm-worker2" + } + ], + "tasks_after_relocation": [ + { + "task_id": "zmboecjxvdyn", + "node": "hla-swarm-manager" + }, + { + "task_id": "pke5uf5okm69", + "node": "hla-swarm-worker1" + } + ], + "instance_ids_before": [ + "1726221ff80d", + "edb5532a721e" + ], + "instance_ids_after": [ + "1726221ff80d", + "81c7197f2bfe" + ], + "successful_reads": 165, + "successful_writes": 166, + "failed_attempts": 0, + "first_recovery_seconds": 1.796, + "successful_rps": 11.033, + "experiment_seconds_including_rejoin": 73.763, + "confirmed_data_preserved": true, + "database_node_failed": false, + "note": "PostgreSQL, Nginx, manager quorum and local volumes are not protected by this experiment." +} diff --git a/evidence/node-failover-2026-11-03.md b/evidence/node-failover-2026-11-03.md new file mode 100644 index 00000000..16e5ecdd --- /dev/null +++ b/evidence/node-failover-2026-11-03.md @@ -0,0 +1,33 @@ +# Swarm worker-node failover — 2026-11-03 + +Topology: one Swarm manager and two workers, implemented as three privileged +Docker-in-Docker daemons. Two API tasks used `max_replicas_per_node=1`. Nginx, +PostgreSQL and Redis remained on the live manager. Images were pulled by nodes +from the disposable local registry by digest. + +Command: + +```text +python student_tests/node_failover.py --base-url http://localhost:18080 \ + --window 30 --output evidence/node-failover-2026-11-03.json +``` + +Observed result: + +- Hard-stopped node: `hla-swarm-worker2` (the entire nested Docker daemon). +- Old API task: `sesmka0dxn5k` on `worker2`. +- Replacement API task: `pke5uf5okm69` on `worker1`. +- Other API task remained on manager. +- Instance IDs changed from `1726221ff80d` / `edb5532a721e` to + `1726221ff80d` / `81c7197f2bfe`. +- Successful reads: 165; successful writes: 166; failed attempts: 0. +- First successful operation after node-stop started: 1.796 s. +- Successful operations per second during the 30 s window: 11.033. +- The pre-failure confirmed message and every response-confirmed in-flight + message remained readable. +- The stopped worker rejoined in `Ready` state after the experiment. + +Protected scope: loss of one worker carrying an API task while manager/state +remain alive. Not protected and not claimed: PostgreSQL node loss, manager +quorum loss, Nginx loss, local-volume loss, underlay loss, or physical host +loss. `database_node_failed` is explicitly `false` in the raw JSON. diff --git a/evidence/rolling-update-2026-11-03.json b/evidence/rolling-update-2026-11-03.json new file mode 100644 index 00000000..d9bf1b30 --- /dev/null +++ b/evidence/rolling-update-2026-11-03.json @@ -0,0 +1,23 @@ +{ + "scenario": "sequential API replacement under closed-loop writes", + "requested_duration_seconds": 180.0, + "elapsed_seconds": 180.118, + "rolling_action_seconds": 115.8, + "concurrency": 4, + "observed_instance_ids": [ + "5782a3226f58", + "d3604e18baea", + "e54e39339867", + "e55cbed4e283" + ], + "successful_requests": 12147, + "failed_requests": 177, + "error_ratio": 0.014362, + "successful_rps": 67.439, + "successful_latency_ms": { + "p50": 37.729, + "p95": 189.374, + "p99": 215.986 + }, + "note": "Closed-loop result on one host; success latency excludes failures." +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..eddabd2eef8d94a5437d6168ff9c87a78ff725b3 GIT binary patch literal 47505 zcma%jV|XRZx@BzJPRF)w+qP|W2RrDP9UC1d9oxo^)3I%LIQh<*=g!Qz_k45q^VI&e z|5VkgwQ8;Rt*tBv4uJsz0|NsB0z&#Z{?7*m1QtX=LS2MGMp2SUUPeqpQB6Wa9TEie zub-^z>bb3QVg*ju^jKS3o#9H#w4Yxz1*n>pYH+2nC3dC@ic(OUh@sI7>n^@O3t+EN zk19TR2&69-M23X8{h9JYx|8)kwwf7ttr>teD4+VN#nkbK$s(IG`^odY38j0~G5LYI zE8yj!-3t3WJpbc*GP8f1Ijv!GZFxNt(Cq4DxZU@%dVh&ur@bEGnl2N^*QNXDgt%%uIzL8-|f9*0a_P$gWq1W= zyYFqsd}OSk24kb~1dN}B%z?^{HGmwKoogz&O?>^nlNT;9zKwXe(^*#}CA6|3JU~$) z84gW6*^!J(I2cJ6Fex`F@*8Z;s#mTo^y2AJ6hSf>Ei3lYhvt>4{wrqH*`8wlt+NqV zs$Y#ZDUzSWF!beISEBi0Dvx#amw4A=5p>tM)l(wMg*GU=hp|-Z=aZM_pw^OegdgFE z#1Jtd_&p~_;Lb@JjM5MZdJErBWf7~hq^IxX89(}?*<2v)uDSTyr#g{7fM4R;@KjPU zef+&aPhcAskT5|z_09<(`3G^SKwJ0e=Q(TjU}<2E7jh(ZoiwT{!}jl%GU(rNo2?a! zx2+TFX}Pt%EZ7ohNMI$bpk|IVcQ3Z2tWLJSZtq)*Im<#WBDYEfci;r(!~8KiUAI2I z+)9QJ;|lF-J*nrrVRIf{Rt}tBY`7YBW#R+!Qox8y99}8lf<@)9znd`>8Q;dY znEDDc?e6`E=jWxW%O= z*dn!&=MGIsRTcKy#$f?nc7NBdssxcHDt6p!g8h@bt@_P63RGK`SeA81RG5nyyn|pn zh5?evjH@qk|v=A$Ff0_lB8|p@3G{6%UYG`sujf7mV;X1<41iQ?RY8pV7 z+JTVijVDHlCoGIu&$AT+dB^5QPcKo%0%E$4uIC6MXfn^S5s%Or=V!~H;WD2>O)~K>|X>YMP=}Y_0pejZk-6r@uWi|+^N62^lykrsvI-LZ#)+m^*{7pxuE$-YJPa6ES98M;^-kOi6XZp$Mun zVh_^?Hp<}gH$rrm9(0RoI9SWRQ6R)wVQt0P3)HH@+_$;Wu?Pdh#`*-jv&l=#aB#ZB z__a1vF1``N!=i=c>_*5tSi+du{D=L>p#AE6M9%CROw=u892xWbhBJ2&ZWOPUu9e_t z`J0llKMY7mQOc(WraFZmW=wk^KbcDk)u1}fF!vO9a$)!UcLP)4H1`%4xgRqS0K?Ri z5wDR#A&14*d%ZEfJ%yaM!xA9$SjkFRTM(E=VBF=fl`Xebo{4H-4hj0}f`xQV%Siw~ zm)X(4E#M~0rjvozMFh8$OtrMtNIwdWI#K9mA^S9Y`%(O7+DH&z2BPw}+FP|N{8`yc ztMq*2M?9lMzlQKSXTlP7_S}q6O5>aSLKTkPfx$(5-5iMGcgSoF6$&wzunij_p=r=9 zULJ3>$)nnNCaOIhR<^3ydE|tmD2_eJi2rKJ>=4lfdXl%T^<`2cL8Qnr#g}u6)mqEfkdy^j(pd_;1LfQq)~T z)#*RRvAV3a;5g%FsE=#2$4c)4WyUl~Bx{f3L=Y&s6_!#gFQs!SM z%Ptu1IMS7C?+LldgwXHRxHrmZ7c|W9txqXT!D^j9u-AN|Y|OWq2SC{L<-cTTicAmi z_r#W74+DHIHg*akRkcJKQULezAc{~%>2%5wLQ>VNv3usWH7RnZ2Gz-YT0A%><>0c`H5JO8&DXi*zR64@Cim$sxd2bU<1bGfQN zYN$wwe1Suk{w@!&Grd0uH@kI*wheyqH}Pu37`unlXJ3eVY_&RLtw?MtwCC}kX& z2r=ymZ+8nA9_W&_-!Uk78%AX;0kBIr^@FWC=Iq?}st>4E&+p_%duBh3kVNp=krEPD z)GOWz8oLGhf-icgv}Z?)m7f&8FU^%9YU6rK!9w3vM<_rm+D;$*BFzlm^yg?%23uAQ z%KeUiUgps!x2o$8_73aGGei+l?ijb$qk0&_pcxE$L&m{m1E)z5{%6fgW`S-VGaRav z!S?Iw_l z8bcI?Ymm-FZKn!Np;!~O96H)0IY%e2ReRm7kG<82Fn{mNbwWMVA4 z>uZy@Ye%>7g;Xba{0<$EH@{`|xhnAW31=;CMC_|9j?M+?>Ej*_aqKS9>ogRu%(R<^ z8J;b1?=_I671Vk@wUgy9Y-KNgni)d}*j0y<^urrM2Uk2lFt7uFt`+!g{6?nxn8HDA z-|mcYugdaGsE%N=JvnV*xpYv3#ROT8=BsCVx@0{J239XjS;u0Ma+!u+Fwr5ij=6m0 zLSvIxxB1C7^gHZ#DcL&5(^lO35rh)6% zTsaD~2LM9BOvklgrH#EyzGJ%@S_@lewSL>+u5R+Tiq+s>wC&&!bZ{TdFdO)hkb5-6 z$JW2#Z|Z!%lkE+Ji(AJ*TFz!!5aIfBcEyHaG53g88ae_isos&=hRdKu{(IgmZ3Gds z7k(3>R}TbXV~wbz&J~3lCtMn+1npudNl-F=qB2KmbKczrin|qq(zUiV=mz!5jQt(W z4osJngz2I~I*eB?O3AP2V$NNllivTjjiDCk>V%*qVl&IrYG0a8ch#hengcSQ0H~+K zBrZ5)DU<3ZAI!Gpd$pCpi>TAd%xh;}9a74VXzmbM7C9K#VsIv!z}_@E{+d_U`?Nq% zi@u}DiWhyB4y$-r=+xk@;E9jM)7*`fPg)%mEu3MTd`DT51r_)uS|F(! zH_LOl6m)}??`_oHJ&>D)Os*^s<836X+kL$G%25M*fJ%&Z1B&T zx8I#PIpI+RmNaLK`Fp&CnIwK8BSBAd1%72kS{KygNw=~bG)!%CA<;1+2uK$d2#E5( z^@|w)w_j8cQIwICP*Z1Ako+&t$S^qx7s8AJvE@f{8IQdTeE6YPrV5cF8dS5|VoNd< zANp`#(YR!CkO|d!PGixcChz$md$u3@%N8#7%MI==THk`Dlx!B6XY2sEDGUZrd)9ZV z`Neo82#w}>2PYcZeDI8fty)z?U1X_n8XA^i5hNk;Ku&STgIxXgtP~=n*gS)-O^6j7 z-R8FH?Zu`x8u%&h7qGwPXFENvoAPODTR+FYpC8NT{G42^n5yv;0}-EEv48O`iX+}!?a@(PLya{a<6*$#GkW&+gS*DX|y z3T|bC+7j^vr8&A+T^EY8jqRDWGEpR0uQE9h$nPLQ$=sk4R$IL5iqjzJwhACddroj?Br~lT+S2>vo<5ZJwlL{#iW=$)A*+<(iFSGBZie!dF^3DNh z?&VkWO=0E?+KTHCe{4{tuuxRaV9(2n@)ICSnZ2(;4v}b^r=)pTAhI4=3C5^0CHG3> z5h}3Rg{iTfU#*m;NN8>F%TAm=@&ZkrpGX$TSo?}+I$VpJo~E7Htc`3-$LXM;rG9lE z72K^9V-I@?9QApE5W?Uzl-x0%^3DO@2O@e?1gXf|5|#& zaPJKC&qP7%bNu_I|MIs>uk=5yw}q;n61oV+J0O+OAx&;v;wres&^q5jLs*uj3x$aS zGMW-4nrUu5pKw|3SGz<^!a(je)0GZ68as>N3*RexSA-RI0-B-a6wht;CExAj>+9`3 z-&b6E=8n}>KTY4lg_b&U0yR3jp;S#E!jhxxcj#Giw&7vWgRckGs5^*u)}A0>LkQ!I{?^dBG~R6ZVjsS)_$H=G&-0-z@Z^T z;gk9U-en=IA!lcEox4>qLD#kR4LdF1skHspDSpcDCtJ+ybScF*(D?T!p(2YlA*vwq zAJ4-kR{A+sw33EEiS2Z`n_qo}aC3;lJcfq<;{niS?5-}rPRGD7m$_b3hl5hZ5!aN! zPLy%q0TY}4dDKRy07;H;-8fl_tf4Q;8~MGZk_=Na8%JZtZH-%UH;0oEvTv6|jv9#5 zX8r@RdYCzRycx0WuBIz*2gC5y z)@!vL!f>yhNfq-Oz{~es!{ua*4Ca!K4t`s7c z?iQ~9gtptia7l`q!C%-Gm`i0ekj*E1s%i;tDz+ew*Y5eDj+TqZT=3(@w4{BmzLsxw z!coPP;`-z1E39lmq)-pBMaMZ-6|l&KD!tY3aLsLct@ZYHshJprsG#TS`shgFPp8V^ zVpn`qovk)vp|y6`lB+%uZx_8!7sE(RD4jQnwOblArJa`ci^wW`^a7L@xC*=OWa6+M zWodt%>31m$zsTBhf2>XGc19kgP`HQ`g8jk$!D5TuerlYMuKnf|${e0*W9mQUHk_Ev z1}3`IX4Nk_!^H+3Mcz{yB=h>ksBn$HPmbW(DR831#0o6v_VR)8<~YCJMRB4}c!C+% zE(&$bqy;^T&;?C?Oe2OLHskKJzIx*E&hoNHm$F2u!;$|m{&DwYVi1pqDLHKXV@l)k z36#r#G4nvPjNrHa_$71n%MJ0`6v*1wky|(>O$xHJ3V1>n3+s8hR;IV+8_`;T4HS#? zDo_Bx04bO z85- z>;Zba zyPAjT{}#u8!EU35gBrRPMj#^z{$d`Qa77h|qZ+tOsx>Oi09Oxo`F3$}U#JV9^|yXv z%A}*E7r7^|M~P73<_q|I9kZF`ywlWE;ry@m88DGWrtFD}gPfNvw_Lvqx2b@~_kB8$ zv}^c&Bc+_zj2AH)7YDT;78bHIoXODzI*o0P&RWg#jg~2pza30qE?_b$U8NSvMOWSN zIHb~7wgBX;vYiEs-UbVuI7t>P33PF2i&FtNo7Ol`xJ0n4`DNxKG0`#6u{1%FJve(7 z6()8&O^z@Cm+@+II!-2hvI<;Z&&BeE79GZ;678KP^0R^Hc#^~cNY23 zXU4@&2L&gWb_*TPRyqoIHurXobs2qgWjGU)o6xR;%r?K6?I~q$f1y6EG_UQOemWI# z;9LlKge2*183ODujxR$J&WdACrinw@RlLxSPDp0TS-stiKl>_9aEFlWBz z4o))x#Y{SoFc;HF37qwrWdsFTPqL2&u$$VQ%699A5}Gdrv*zqU&NrNW!e4V($Q{Eb z@C3EVde^8R$2|_zL1pX@TNlTcLk>Go%~+9F$r95adgKnGo+d#?3nbB8W3H@vIViDl zNdLBOVr-}K8UauAi=yA$+Vt`1QsscTU*%lr74*hlOAW)u+*ewJHiY{qRFpgv-n!X6 z_;zwk`r8TBo5iM0`)nEPT<5(udKJd|fU~VoZ#uv+S%6UAl6zHFp(6!iA&>i7nBdwR zVizF<+MXpZIf(@zQ?6-PqZUpc89rh>{hU0^U+qm=&FW4eMb^^A)OYC1N8i_gZ2}ej=NJx2ZmfOOT*an@)`UG zxzAZ?M}$(t(9tj7<>m>lztI~ccK3!lUGWUVI7hb(jhyn=Db4=)<98-}DD~I5R(Hl! zu=tcAAoSmzYk~jdT+2B+c{%=5ivB51YVIcP7XNavQ#5tFFc$FEsg9Lp)X1_y&>(5_ zm}QV7ql95XLL;J%DXim{al&MmWJ;wyH1ssGQJ^snbuK-`JJ)2kkxp(l7LMsH7%l^D zdcEGjpSO^m8N$PcF4Z-{ISCPcj;hrT{jGA}&YigL|4irl!)-zNk2v29MDpKk&YYc)@pPt9^quC8sB_uIJ0dnBf_RA0`WT0W5c1_q%Q*_If_syb&1MJXv$6n=m^YAi88`5kqcgxE zHT!&IrQGr=Hag$yPP;YB)|O^{4_bY7`(em%E`uHVCOB7!?WmkF4aKx2aAp$zOmB!p zZ{sfS^NgnElY1m^zXJ#n#{EG6N0;{ZkMZ|66Jeu}P2N)0nD{nw+2kXv4NQtveLTJP zq8xB*CfeD&C5mN)kXl^4Z4P?bvd6J>2%aY;7Z;Y^%^s1COcy6RISiB8O=1X*RSx0i z?4}wxXqs&73|pz8<9*uS7g#mPX20_4GZqpdnl>m(;*1X-$>OpyqLNDt!RgaVs*FjF zpEdmoBj7RsbXIlQ0&Fe$z?xT6@xUxtvMKb%c*>wkiL%Dj^2jlvA92ce&*EpInxGm; zhH7{Ec1Y!xC@645q356GIhKr&|SNuCtFCPPwT=qG!zQi zf8Kc;@8L|h@U0+?F9Q@wk3E%0Zb+P*6XKSd7*&vP`D)qZRWD5=J|3oAVIByluY?Qn zaTzl&XEuTzt=Ce4lfd5&wESsarOEY)V?`&_KC2l(j%u31)GCOuH1;Ey!5W?7Vj_Xt zS4OE7xrP9fv%yJ3Omk^Q3YCS6uXsHEbPwgAMwF8pEvx8E&(v)9uug;#Cz* ztJ87q7^qM(UJtO5ooD%#E%^OpbQdPeZPr+K$FX7iabuC5c347c59={zQ8Pe>{&Yu9TjzTlu^n?A=u%gzT@W990w?gVu|$m zb(Z2_!!&KO#AqU)nWBhA?z58zl?8E1*)_fFj&LVdsmEoS{}-HEK0Sk_U8K$ROo zS&f4BHBE!>^LlF4XqTaHz5IW!xN}gFbBh|v4AZXIDft4ciGxcpM@8SE_G z55YtvyO15!XNCpN<_;C{#Iq8G4&@}mG`yT6VdunGQVCu)$`v)bsaLu+KjYuUhhBT!VVve9>y&;aOKnTM6I zQAh0so87UGBP+SH2bI&$iytVb?!=+KN91was;F!4qK=`f&F&b|Hhs3ne?s#TT5~1M*dJ28dCgdFYel( zZrY0cdX2WeD8+cJyDiA^@2KQf7isPl5`DoEApxq~m(&%E7-A{hS&M$ot5)&h%5sxZH zFG{L0Hlk06rNVQwz+^BQ1Q&~W)Hss4(u%aMV(LR}xhAKSFIId1iF`r3W8z!MhgYDF zWX+MoA7WuOqlsBFj1^Ume5Y=-W5z#SmX)!~?wh{-G5-k5Pvg3GcD5w=P(j&|5Cxt3 zQ=7lm{+K-Rt*>7Q5%U_Gm9~E}Fd-}C6b|$H@jw~YN^vqUg?=a3^py$hBeAEZ3uR&5 z`iBIz?dc4?iGG0`(ytbziHhj+!#bJ1$lG`d{#)>B{y1jDz&^?6H038ESfDq=J0KJYh!xV`Y3P4+H&(E5bF*=@`lpJ1hDHCAgk~pQD$NPw z40kv8^2$=JVqga4V>Y}DMt_xO#v^^U4R(PdzjQozP+vKn^`sb*-uc*tmvR5nb%lHt z$12E>4JsB4l)I>2ntpuI|GX0~T@nj{(&vp`**INl?1pR{9K^<_c2#B)c9v)6to|Y- zTFI$w&7rhj$By0lmKV3mUzWbww+8#{n8)PRf*w)6ak{9#QSm!rSWJ$dqteIpZAf|Z zm=B5J3{HqdOV@gWVQP};g!q>+g6;U}ONqB5U-0&~L$6bVT)o(`%vb}XTm3Y-3LClW z#FuYZR))(W#^V=~Oe@=J-K%gu)ELps>PquO2$5v{@z^P{hJQ$FQ zA$l?64|d3fqh1D<`F~*mByhB0BB0Oo`EEMEe{eYQfkE!AnGki+tav2&1Uy+^%* zG}A7CItGc{VK9gMhRBrXA1140{mLRP3w$R_@CuHf%Y4HzsSAKR7WxaR_N#TtT%Rs3 z_-|d@e{|dX-w^dOakcpOx4kg6V?}fojCaP>hGOlpFA?yuc?|2y!eeAbXzX7aQLHKM zkz2D{8Nk`*4yG_jCDAsAiEXvf6#PMm$Gl4*E#!EU*7mb5{jEB?KVF|8jp5`Fa*>gj z=7<-_mOR6%D%{F7Rd>rZ>&gM628E_nl~Ih+jA1k_u z=u5{EPfMh2N)mLdwXvG-D^0$0FcOkVX;m0nrz7j<$Z+akz(G17T$c=`(evW)Hh!Q$ zarlMhAuTZhC)nIuRsn3hF5u4@e8`=~%YgQgE8baxjkSO~0~ApA=b2bNgeufaB5^Me zxIU4mtw;7wgmo+-YPd1AwisWQJE?j;|F}|l$22wkYWA}m|2u&YG#%H1Nb`yCRdX-Q z0^g-5rq~HhtKuB8_-9sa?US06o*q6n^q?4!PxO zu}BX6CH;PRi=sVfoqm_Y5asKUxNsbcqfXGgEf&o9)8~}2N-VF?167OQ2ok&=^k}2F zGuwwF+n-g1m*lilI2*MHnW9%|;~mv)d{k=h zp)ERiuitw}7QgW$4I}CqDS~O_p_wBO5h68aVzYH4HCx#Mh^N z22nRj9@@2gd;n|78D!eTtXQr_@BcQ;F3i$A_gksSU;rpphqy>tbuX{`@sHI1&hY0> z_vewJgZw*k=l)L&(*M^RDJ#fQ*2MWE-69f3CgR(HCNthHdih1BKh!BMRJL|>HxDzag_13XftbrL zFx*$+GE+!4g=`B;L9`s=Ze`uGbex#B2S+_z#g?Bx5e~Of>WecNxqnz}X^|UFSUxb& z$d(`Ti^wnj2sS&TK_J|B3mhxuZmi}$6;bG^o=(Z>)ck?G2D-)uKLog#lr;S z$`;Ds*e5gAmtAHxs_t;uSO@AmP0cLyOrJA$h)6X~P4FzVAt; z*c%;$MBvgjG~`{2tGzq*S1{zontJgj_F9pC`Q(f{pU@~1g!EDUMT|yoH}+ni#RZiH*5Cb7gc>ThK55;4FizECc1M}Y7 zIO2uDEXmcvPInW3nGZ`8pi{@7XKkDifh1TTv?--O{*mwEMQP9a3Yh_Yw{rQQTfOOP6oi8_C?I((#u)D zzCZp>l3~Qhx>fTjB40m!GE<@)dcH|jK-n4fH-c(Q5lKuKq<&99@We75bCH;nes8V% z&nYvMxdwK|4~YQZd^!qg6jxlRn#$VgTK;g|`^I2Q{qZf@YPRI}7$G9aG^pwL??VsvTkc{0 zqwO=^zs}{&g2mFZ`FOrrD=FijMlju^$+Zlqc^dGb>eJD}p4fl=OLGcPN^B1=PTY3)_ zim|pf{}oa7CeFgkAd#VjOi=Sg&F~KY7Xp{eK~r%)(WmpbH38QDglGOnk5v?uz&;tK z+#iPQ$>Xm6`YY5jI2vj+bWBbJ9t%;2hgWzb&_TwFltmIf2#pH;AN55S&lot^NCnDR)(w}I+N)Fq5zwHj< zRhHgfZr*OJd>^S-4BQ&hnIeW2@Ku31yy=g{A7NIMa=HpCQ^~`|$H`y%@^HYh3wuP| z`UxZFhcFtc3~7vAnnxU!8x(Q|k2dJ`sN9WfWjM2YqGvVpK&~+~^M5>{RZ>>o8%2tu z&E-%$v!m9-*FHi1wbRKjDWl&$xhCpwxkl(e*=Y?&yZ6z(==~h-wAFprs_&sJ5tp0rb{sw;v7C%E(eK7;od&0)DlN_~Xdm`-|Jy(7)Wqmk3 zXCr0Tv=_<%t)rK~{_BNeLdTbavc<{7{!>c2J#F>@(ZL^ux;i}lm+bbLV9=t^1G3*_ zeY*I$Y68!}&7>W?5r2L^Ol82q60or?*#j`JuQxSlOuMw$sWWJG?95`jK3BD0`Vz0- z`<7j?H=$k$Q=nRT&tqp%qg1GA!-ZqQLE^;b11&}l>?j~Bg>evf8%g8S-Tt-;5Q!e zq?RVbekTKnK%DO^+4+egr{1o@!TpdSjUyBDPjQ6rH>NhN+MW;f@3(6b21q6p@!^xl zO$B$zdn+astC3}b&xkB(; zeuI6w|9XMzOJVUKc=Rg$k`y@%Md!n^l$^fxCim(6yFDIX(i6yN%%-LcoPgg0&iRqy z#EQzkO9M|ct(EMUNby3__P=2;;2}vLkpBX=0fxG%)hDo9{?=jqeWm`N{Pi!|8K9x( zg|30|jwF-L4v|lT9U?Ic^QE&$1+J-KO_W;ICP@~aLpi#Xt#lMPD*q!LsL2TT4DF9j z6tF$m*zuKSO!w#)li(ltS3=##boOGcHchI-vp-YKkM9qHe($fB&1oR9+jIcv$4jG& zZuHE9lS<~sWnua3NRMIl3jG!OiDB&!~G>u-3t;Kx8_1xTR_8HrW!30g~OR0M@Ht4>i|X~psU;366z&XCQ64|PT^JwiqOMb#j;qn4m&QcY{aY)&hYvf zD|U3I8KT>p9I;^AqYlZuWD7gUa9d2Y-Lxij<}%pa?%5Dll|D1$9Lp<8-fBQCe0zv8 zun$=OO-#fN#Se%k3d5H<6B>Z`h(ZatK5~!;m5*F=P6x<`L?8$6v#QkOLM$TFCT(zCtaEF z=)g!t{K*P}r#+1ejMSAQN;oPq=~qi!dFW9!?6iC)mjJe=D!;sH=Ho80M)|bU5;qxo z<}(9A^-glAVl^neEeF~sr4T>_59 z#VlnluCJCE8>M61cRdZ0a#M5}$RG&QjV< z`cWhti_T3O>E*qw4KZ`L;ifnuw2Gap%keWP1eF9bw@YkVRjO@fd?^dAK^W~5+LpSp zLvZ?-HDa}B+35^dq3}CZr_z|oXe4nN2MX6BU4L!srpV1{M=)x!kHyGFsyV^wX%*5& zj%l1whJO)b&x!UE3iDP1;c26;UTDejL&INB+KhA2c_u_ABi|LOpJ`R^s|a$BJz+9H zdDPgMs*=8il|hd?aR=5yE@2g9{K&VQhmkHU@JJ%kWWHa~N!4RqD2++I^CfXu_5W7= zWTk3tCGMpkrLQ||A!_pX76azn6v}?&m zYQ-f_=UbI0KX#;!e4*5&u6Y4`Ry>`fcNUITi@8o*gJa5aeV6c`HvMty;0pyA@p^8!VNIkd?eT4z8 zrD%+4j?eWy&|Bxv6a&-d;v%xldoCgfYRqjCk}?BvH?AH8b!P@j57Qd_R#(Bdr(VhW zm};qlY*799f_mR`Nj|{5-5~v7R{8DiT7CkW-;i_3@L~m|c4&A{x14}s7ra02XJ)?C zzpdZjBwyb3HeFshSS|Gyg1=iW6JIY~ZJ0$w2>CYv-iNub*?5bu%@L2Ktlw9LbLijh z!O~yh5%x>T-bJ7KCH&RJJX!g_4{SK)86cCLHl8<* zbAOBC0Z4HJWh(d(g0{3%`xHHwt_Js#ii6sMiUyjtK?SBo|6nUbGvqJHrA0X-SUVYs z@-@*|t2%>gi_-aT0C*cn9>b+Qz3upmK1k{Ul=|MbXwGhz13tk2;#plrA_n+RONs#d zcZTJE;MsqWtNH)clJ+k=o1$T$g>QipXo#i_^DVVO*;-<@;U)pubXTFAK}q;>bQD>?zpq^&Ue|EEJ0M5QW-U|Vogtf zO!qm+e!IUU4uoKiE=4dB@Z%JEplBeo;%uo79TH1lP-ahNarMzia##SG@rZ4kbF~gx ze4mT6tH&I#yq*APjOgTFYv}y)W>20Ta&;9fi6YR#478BPB{OoXBP2E+n z3(*@bSTGA4mBAV_5E}6`Uv787C%(r+3VmTdF1&t@&mTSi z;O^Ba&{KmCbHkq{yCc=WL?r6AC3F2K5)y%d=IIMAEbX-Q_;TH2CW2DS=me;2R8_K#I;Xzzq@cu=| zh0fety|9RjuDp6b2*aRGT{cAr4Z-Tg0ZTop6hT1ZbTv*v#et}ST-Q*Fn{EV)yo4G( zV+g3hbC`0g0B-`H%Tf5%K^I2t&NNRPp0m@>`nLc|h#b5=h#aOX-I_bSbs=ctk;gt| z%p!@6gXs*CNlf4iqMpeVKMsV5*`e4wz%7j61` zci7{#@grsvJeVFnKUzsTcy)y1VP);2Ge|QtcOl%o2 z`;5@w*}C%tJN!cJb&HH*`H{-@JMbM#z^#Q;aNK0=LFveZeaa}Pf{@xWg+!__ClNLG zf4!uz_f%ieC>cz?La+)-i~kh={c$GpMkXp5+OrIU zDD@HaU132fuzGTT&236x&96I1unQ#1GYeAuK9Qo)CS z$Ao}+Qtk_mhWvN6a)O|-*VZvn3$I~y>cxhnNc7o(?bGPnuh~8#dVa-^TtZW!z=2?y zVl|HKM&2sV;XsKuVYv6YhCc$9DD!in30aDM8rNFbJE|o|NculXXE(U8R=+Z&tz%y*@vxc;%=?( zYT{|(>SkguW^G|+XW{xUn-!z6?)K?0KGw089ooX`{pG?asYBTv#Ho{S@==5fZA8H4 zjT_e-9g~VP*Dbu}R8cXyuadOF1+RxHcI0V2CH>v!u{Ztaao^#r*mK%#tjGAOO};}R-9`trXm}wK zb+kLrNB7I)NF%jg98;g>lgynW3wQwI5>v69Akzw&15f@Hp<}6(Op4%S|BX(r9Ezir zIZ~fiZFK${8u6h`CSUR0&jh(X1k6g3dHX&;qtc)*)Pj3< zyq;oDt&a&%KK;eVrInUIjUXB4v`)m-o?^Nos`Lm^WaNz*r=gFvzpWVUOAQh~LuXU` zQaoG;5nj5Eqpqg9NLD@r3els_(KG88TuXNT5G%b}LcSxKt}A;-RfR<=)^tkSJkoCl ztdfZ)gQVkiedHerQ$C0^?t?JRnet*>AHFkyspjqago(grubp*JS8jp5zvTjm zIm}`i&UQxljc1l)765=_10+O%uia8}i%FTnT22Pf%t@&v5?S~l3^=Pi59LZbNMR)?l z+zWl7Sk%pZPe|u-vLQ`3eaOP`-R1BxwTL$Hl0>L!;WjL-DHn4d43wU>ivV6gynYS+ z%wMrjscs64!w&|wXFR$FzK(UVuHD}r^{$8nNd|zEt>}Hzz`($Uc`RKfnZ~%Qx}si% zG2cGds0;DD9km@-02b#tylF7c;&kUPe{sfr4W4mS@P)C6D{@$d^?l_dt5B1qH7|F& z0ycnVqQBx!jr5BAM;eu#wm_KBg~_CYwM>8kVrI#ep6aH4|02z6@_e&I8Z_H-PJ4KE zSpT`0S1s%BoQtyjUmuK`UJa#}TbA_!)M)Nls&mpywWaZyB1-w)w}RGUO1mQg1ZE>M zhjIwbbu<#qTDXA&?=Reg%3^_6j&COAfM2(q?T7L!aBt6l@Zi+AN#-g)(q}j0bMH3` z$Mw{fJ)7U{ZccIa=_ibffGm~RrKGmCwk_;2EMuK$G=RkZHhh~`5rJeYbL9Y(ltXyl zaA5tr6NLtac6Rw@WnRpMJD7)kWEUoZmUZP}H_J;Sf&wDL752lz&#C$@)nBOeqCIJM z&0%LYWY#8JGdFfaxUL-%lh2TJSMlz&Lo74aXbAPB5S0s5-Tn7|PqhH0yXi8qP18v( z;rWdN>c@j1(7!AOyq@tX!l%XH{QT30e_s<3`G2+X|7Bf!Co{XxG6>V>a~FFLHyh8- zu3vi#5i>IjH#Y?nM-!|6#=#b!0X2pQO2A|w0zDriU4dc81S}`Lo3J~inP^0ga3K!= z{cuNH1~mqC7LvnV82yg;tGGdC>c_3-}fd5xjqE+cR@3 z0@+W?15d68AKVT2&6CgvpLMa~37Bb+;AG5---?ictL zTRtwRS=$f|3IMD4kgqkMJ+biB#|e@9lzG z#r@ba+vb8^yumT6UEp7R{)h9_1}pW+{S;!lzm2c|l&7i}-xsNYy&%QH{YvGn{Oy zPq=&X+myyA+I zAv=N2kQwUZt4_Uopz<7#*hGUQdSPnff=_|Dov$dHy(4Z^4!0vs7+7-~L(V>+O2rb^ z5wL~3-;oH!G-IC;as^a0h3+E|troGdgwDC0fUeF)&vYWtNhTMRAYrzqu)BWgzX{05 z{|$|kQTTllTTVBYrKNyj7_6)xPKcrsp$9$}nFs>>N=wrOkkZ5v7_llvpJcqciy)?i z*+u3ul|zSHUM99<>`~WTyyz1QZ-KVRDviZ-6eXKg9pmY>To&pv1bhK^FdQ)#>G|BwJ!eTd{hJT2-=pKbnbv-;k#;R z7^r3)5}6rCDM2Kh!g**LU15ynMg2NO@YPt;p|X?$i70BJu9QKRRmvJ|g(eLD~UeHUOQ?CUCU1e`G4CE@Dk4rdOZ?r3fXIq91eUdt^e6;A5DU&_+sM781%z1TKeEq3d#`tAgwtW;Ya`{b)Vz~!WyS2Smz~}9MA3v zR~!WPkSc>L7=7%t5Pv6v9B-+dS0kyIf!{&jm3Vx>y4*8Eiuh0H(o|a*%p%&`L_Mee z+(mwJNTSL_q+Rd&%*o=>zeJEI+*{TXC$e}Qa`GY`;X%=+HoPdSdyC>*Mb!k+mHjAU zB4X*fYD{aBYHwU{dWP758w6({F&Q7JK`@X%N9!o7F}iA#@OD#(1j2GS!@qLXMX5Z2 zEKm^X@IQ56Ju%?(X0FX!5oq5)RmajNiB*TM2p2uk9((z{U+XZfDndv>kVCrfke`~z z`&)MGF|h26q%dYmaq~Ec;6k-X=QKy4`db)VM$HvQuHILAq2cD@BxL%D@ zdsGU8a$%Pk_c}o8lDnXCMqR#*|S% z!ngc|i-5-$=Sq_Aj8sRg0XNGjjFVLhh(nnEYYicDz?Sp0z0T~#j2f-Y5N;P?#puBc z?$~bt(?8Pnaaz`KJ80cy!o?~D5pN@3Y(>%zpt$XKZSVTqRK-Zh}rK2;J$;5~P*VP4G6sd>hdfOz(NvIg^jyz=U`Y z4ekZ`{@|oP)`DN8^<&MHi8rC5t!pmr{ylHMVW?INQtbIA#aUFIH{Ld*f~Mnh-A@JO zkc$+xpK1}kDKpLF4Y+7DVC%>tD0`1>0ghhAeG`#8xqI-7x%-+HVmwR2J_iVCEhPPu zZ}Fi()c_KiK!@)Ti@)NYsF=iF<-@cu-|a>OeJbyII@cc5=0_ADx(SnZ|#Le@Wmu$BP8{IY4WV=p^mVEy}%M2GSB{D&!Rf-@q-#@)w6@|CzBF9 zSiQKDKvcKSlsl394^`g_A|P|rcJjx85)=<*^;=!OX8pXszN7scIiq9OQvtc$NPvVE zlGqFbuf9sJ(yTEuV=Xr|xEDdXg0Q8a{4bZ4@>Q%2Is-s60I9CsUCQ!@4uNxSQ)w6+ zwRrVzsz=DtrQZ4Ko?z7F&sW;;*wlL_Ph8HLuUB3>;KM+M1OI6#XB2sl@cTg;Apa3* z{2KuM??o2=X-2WOGcYC=HZe7Dv3CCNxxAyDnd1-sl(ukkHnIJG0BE$5j@^PPx-V?_ z{)p3h603X}ex;a30xL{a6AX+sKZc^DdMPN1NCt4}Q~3UJJ(FLP_(ELCrb9iIZQClX zTXa)OghbyG>1FN(-bKVE52z>`1G5zG^{B_z%;?wUaAN7NX@m6LOt0r-oI|zxEwudr>`mae)!t`LISBexT=#i^sNJJXqZwsS0RQYC+o?$*J{}5 z`&}fcR?FB1fotb4vKJi(E2skE8e9s+H74LPu>M*1ouJF8PBD79far!}&!r2wKNlJ(do0baXaXkWGtz zRJRU3>!<*O!uK>F8Dl2d27Lt*Hn<8zXxpG4z1i{=g`fP^T)x;}(Fn{%Xbg3rk52u! zStmvziNkKp%9>{4C{pK~A#n?NhQg!28Z^k!m-2Uc$`4&m(392_Lb@I+G^zF!uxZ^a z;06rkBdDl~A18}h6G9D6rZQ}A$}0DOlh1BD5!cLXA$*A z2AD)X_5CLxGVIy{A|?we@&lNZi`qV57sU`+M(H!~v3u9lKObr+-tIcp7-sB0WKJ|M zL}cZ#+w_dl`Bv7aY|-JbdzYr3rs|{Ha^}2HU)hB1 zlt3Gy6OT+*V|C@OpcAx7N*9Y0jZ0v~2a#klF>mf>K0n z&t;4p@ZM9C?BQ*g#L;^=KNPGOnxNo%V6YC)uVUd>Pv30Ot6_(QPz`z*qk?ES?SJUY z`yvD#$(!~PFcomP7jMHL{OSQDK;p?43YKqz&GyfR>Rl2x-y<$q~x>h6)0f7x)k)~|ABym(65 zt_S@XE)r4}ufCWksFaUiLlE-h+EkFZt1kfDxbdvUW3d!m%84P15q`?o{)ri{3NdH)wI z8?jYqaB#Z?;u4t@;sD-oJ$?B;v}U(xzeGq(oxwa5_kt7plPwn4295C%kc947KgC1E ztV2eAsRoxqJ&x4ud5($!zHBX8!%#E2;)tvoSGV1&QXKJ{Alj#JzcWYCasZt%v_**e zj!ghd78DbMeGb7EKNbj(FjPhGnmXR|*1VAcD2zj^Fea!&d7!9WnCEg-%DR!To!HPx>;TW-g7U1c1I1VI_f5@N5<5Si7UsSP0uP02gm*qy+I**O}9grWeucd^DpTstbRXRG>bZ<7<%*_*z`=?=(}WV z?kch13Hn3FEvTG75{&0d09ztoOhBkv$?*1%KSsnkLm>E1$VvXkkR$ldA@^?ps8}89 zx3Zbb7wqIkHy4)y-XIYWF$U`p^?Q^6AO?6miNR_eb`d#v(=l5(cGgmV_0V|JbqgO-DZETztMUT#I4z~ID7DpV3Vh6KD2%g4YMMvp?XA$&hs{-L&uf{7r7^Q z(&T1DQ15_{!exd^DiagYjwfsm%TXv$OW;a0_GK=oY2?MC1|p)CL992_mGxsnH;RfN&$K%VgeV>M9G4N$L<=r0YVe}eI~xhpgy8Cc zOwAes{p*g;q&Y1vTSJhUEmJ~nscIlvC{R_18mxM#r7rF5hMI`X^yDdljo(;6>d~3N z(M(*fny(6X*80)<4ig=p0@Q<30L(_!_{?5f{hSF6FkRC6Yv}<_j(sKi(T!I`5opc& zI=@;Ak~gL_8E2;(#{w?ZgQ;cey_Z+F7;}I*=Tt&rH%P&IiwdAOc(urCfY76`n5W1r zcVI1(>*wrp$=ecep$(C)ss=@cL3*Mhbu|sj#y!TSQ8$X;Tc+p1lUts2RRaCh=3AHt zY2S_EH#^+0T9jr5m-j7bHybKJ9q3$v_47i#JiL38Dc1xZJb|x>5;-2TWN7X+GGaC2 zu6Zw1rJl}7tOCLFYEXSQ@Py(2o?7FFk!)F$hlw(uUi{X-|0d#x9s*|5o#^F3^P>M9 z$c`+`gZZ$m28J-8nB5)H&!kCyz=_dLCJTFLGdhHDW+jDw_| zzT8z!+aJ0E90wG*`bTTe=w4T8HW zeVF!Q@(43&hmO`W&>cs&mMS(`t!ls=-tNJ*ckF)YJ+ei}%&#%@gF^r( zo(tQGSVkehy)x7<6aO#c_N^bL^!1GEWaB8;!=0Pr)DkBKdpdOkGz^ z7C~LDI9AbR#bd5n+jX7#9b&v7YeHjI7Y+dq(lt_(u2xmQ0-2X~qe*)HGuVwTx4;s~ zxa`sLZw=IkJmVIr$P3eIfPTv;f3k`8Eyps$T0}PGdC5a{c)Wp7pcfJEAPfqg6mia1 zeBKZ)lt+jDHGp-4;8=Fo{1apHztjEw-9dNl)*%DKtB}G(q`U~i2%~_|BYFrgP^pAySYkZ7C3y9Vp^NL~>jq;!-cE$St{(I({?Gpm)oH6q3}FBQ$( z;5$t287%sy&2PpYe;Wb3F?%_k-nPMXWB-!U| zsD}%yD0hhcr7^F0bxtak@K#NJ)}j`6phBT*P-+z$W^#Y0O+LSEl3koj z0Ap7-#_zRc7sGys6;yfRDvE4LBJAZwK-yPoL0pDpU+3#A4?|cx)2>)w( zA89QE{E1e9;UtyuciL)kC1s0@wJJ$;6{xO{?Hixw>t3c=G3{@C7P^PkbcCtT?|je? zz%Qk`lYUzuzP!O3^yV`=k2)==5x;#uz@L7&ModOVYb&#FBhni8xw)pp{G%r!Tvw;K z>>6Ntl1O*_^lK8R7^O*+3t@ThjKPwj>(DQYU*;c@v^eDI!G^x?E{h2SocpqIF)Qw#oP^5N3Yryx8Pa#q_9hZij1Dkm_EJxL5hQ;V>c+zG4*H8dYXq* zpEytZ?0u@8r!9(g=WE&6z%y@-Fq|2ksJTqt9yLyhAgo$->O!j7-%}Dp>ac31X7GXa zYI^d$W5`4s!?RUGvekTT^jNu6Cv`gOW;og18O^?}Be#aGczMeMF_g8x&_;cse{0wG1=-~tSRKgO=ct76udVWRm#_!U2&xljR94c8MlMpA52#~sSL#RZs1w~NiHE`fGJck1TU zE9!uCRj+G|H$!StbgaX6VC8c0xT*kPzVS7=(d8!u>xwogc>>K7JzLQBRyFoWvbEBq z2)&*9ngDFF>OCnAnhfbu?!CKS@i8R{wF6Kc5V-5pySN47SAQ*txc zD9&_)1!2p2$c%K$8T>keaJ1jr@MNJT^sK)b9vlBGV505^ENOc2W%ve8G&#uZY}N>Z z!f;HI&b__j%RBtaECh4x9ov}3a%Q!GlbST@Nn?-wO6Vrm6UFh?d}P33b+De!XM4P* zX_#x%(bH>TvdHHDhd9GX;rplBkblXoOB=QvWHlX_eiV4Jyt)|>xq=!g5`C}#kXpxN znKMe;Z?Y)4$R7Asw>!{b81!ktWSvxZ+?QT{Jn*%=RL$T!*s~n7+_LO&!4v9pOz#f+ zKC|mDfgRNxy{~d?|AJjr;toNS>Y%f-opf-4ct*xBGPwUoI`0O$U6=TJ=NaF!#2zRL zGjMpJ5A<2D(LcH`cGg563qfg7;j#;63;P8iKxswN0IC!73VuQ+a@>v#2&msZ+{*a2Zin%!j0lASc8Bzk+1)r zsDouA38LvlZBSXNbn%lmv=}+rq{7C34mX7?6f0a7e%P`s^amO!&mf8GNQHE~_OXhP%m$R3>J-gnfUT z!Jk)605keLTu(FSd}t}RV?54EfAbtQdCA!wjtx3c36(PB2{#W7nWF8kpd)rLjcR1{ zhHxRkSlZAJIWS-RyEcM9&?=a4cV!mV?3P6ArjWH#lW;I5U?l;GomOaSIf%QcJFF`7 zByCW8miaAP=7l~F^a1~Ms1}}pyNyf~Q&Ydy2c_+*R!bQWTo~RV3UaDhlw30>9Jw(P zV9#drjd|ss*5Vbw`U>7E+c%_U(RgjISNJ8X)Ph+zPt!tYna?gf7<{neajHG!*kOV$BWdc9pJcS|6^D3Tm^jYp+&rB(0ErnDqv7I-#1 z6^P|h^VwO(<1(-jd6RO9PCbrze_c`2^-G4AB6}@MtWkb;cK*6-(3VQRIr-9zAy6T7 z;^;>ZSAtkiqnk7J#$q4);DxG>SL5c&^#wCqE9RqYr;44c>$C1M#(4C0m}*7fAHQ*Z z+cw6qfCmhrng;Ja`g^yzd+NBHSx`yneXPMX#F{+yKHwV01EvJk%Xn6#zhCl8oHB|C zm}!ROZ-hR@C{u|!jMS>MR2n_ll)Id^$n?<|mfY&12MjCU!Jp;nuRI%g_;^YBgt?>WM;M}dvzhZ(qx!RahC{OL`6dOw2{)k*Xbgg= z^{=2vVfhRjw7-H@oi=EJ01KNf`AdTx`!?s zifxjbfFKyc|L*kx-N=LgxVut+=YD?QJQ9%P#0SE#< z9a!==`|p9NNf|~)Cbnt7A)IOzE3Hm$E2Yaysr4pW6^adrk_b&HAIWN;<+b58pT=sP zi-PKeob7I=F=Ph z`V3esaeWxQOQTnDAISDrrkGA{^CfFj*)o|4m~?w4_IY zBYVdh9jtO52)AIAq|dI?w^WCyy0!40#as@jVAy?t5R1BGbU+(7o*bec$QsKd&Jraz zXz-;mUx`9VB*9cljMid%PoKqZGnpLG4q$Q<8#oOrC~9$WItOl271jD3%y=}I7`sfF zA0R(z=2}N3tP!Q5v8Zjsr4Y_!Og)lroovfm1GMC!%Ct+oh&f0c21fCKpNkPn1Z zdU27d=Ruh+x^NZ!lVcTmKjTYUCZH|pZdVl>`ov|%uEmt)vnCA+RP)fjruM+AMzQH1 z$+kFr2r*;WEOnbRTNoxsU3OI296njLLfT7VsO53<0nUMkp0>5pp~if{6=ibhyW;ZO zT*)C7R*ag-hSoho43QQ7GB3C=#5HwJ7d|fg1Q*L{)F`{562zwc&!BcAX-m4)$D+3ha}ldYQ6mT75%;fYvxLS$V@F@8wa~3wUBzL~0cnSZ&T2!DU$` z#T6QIa$qqqlS-(95z0H{86PvLc2a%G$>gdyx|;H0(8jpYbA{{l~;m@Vcf0RKD0l}ku9DGL>RgrM}L;;zxvSXhkQNw&As`7h0 z<>XMk#l5jYmZsD7qCEx)xqW1jw+)1r?J)1&=RLX3jz%#VD(p>J20YxhyHM*MxUFat{F?n!b2wjwro& zwrpXxSRH!PG$aMt*4NzV4eu2NFuO2yre_iFjuQToRXOI1zs2+&XY}mmqSbMc z$Ohs&PAy9~=%Lviq61-bc|k!75w4($ceuVgTMlS{V|%Io{e^R^Lz6;w`J_bsiwtFn zIL(NI_v!g1P<|B;LTZbvCy38MbfS)`Y6IB}Vq47zvg^R<7VSf2B=?q7sj>%OA#v`w zB19AHYnl>fRPgL8dh>#*s!NsZE4($RW(&1vgEO7;n}&aD_f;&C#YBN(iJf4N?tnN`;ixQ3$pv)KE1W;Bp)j;xP6>74 z!k}E8R3m97A#Vgnf<0v=?}=-S!t-&EIeO-q(;T!i>YSOZv?@+EBODFHk5{|K2vVq@KUoxkjo+`?w64uX86w$(`&t>9iy&^Kf=R_*uy{C#Ne)J=fQdcjW8YGvg!NQeMcPc!#WiaTEUrwo zgfsrgJUOr{u?T6`bdmgFU=R+wDX4iV6>~e*EZz9w(NMNe046i%(Nl zT|1kPM|!>>J{F5XQ=*KPp2!-LghIT^tXiavpit!qd&}Sf&4@lIcz=4523GX480thY zeY%u%`fh2TRmOE0ygXyIR$V|QJ}hxLL+V`gy&ZIJh6Vi-`lfNBPGE-kEI$B(stk;^o_0_`? z47NlRk{KQp2}>XBfwBwTU%=Ho1TW+dZ9c5%7qh3^^_`ccdvh5IpRl0VaI#wZf#^F`h)cl_Pnuy zw!6M*BvxmV<_Ne4(Up)LOKF&=gIRurnz4zz5zPU?>qE`czOrT)Ie2Igz2Uq9W#srG zM;&>?Z8-2Q!C)yG!A&G{rN$jX#`hHg9_C$+GQrS)oVjkxjMxr|kh|{w(ASd|0u4{T zJEQruq|JGNXQ=%uPaho)D+wa*=&XZ6E1;a~#=hX5xL(-vY6&W=AZ=iE$bRSc$xpi~ zq{HN}Q*^4eW;uLIkba&`m8uV?=PD5H6qGdi`jjI<3t5Lm#4No{4z>*^WPO8CSX(TbuLe$U_YKEAG>?g)fsZ*B zbpnlL@NACLkVQ;3Z6TM>B^5$edib_k;C8g!$O!MWN)j(3qJCp}qr^!##m6yCJ_*|W z)+6PxIKy;1%e=Kbb`>{$|4yiT7k-o-Jz?St&SXKIv@s+~n!-SPbZWLt9F8l%K%5k=pV`YFF zb4Z0~CF=o)$>aOyWE;THs^uo)*ci!Ndmy9fy&9&bF z>XDhYhA7r_4z9fclE0abEorHGvvekJ9G~OiVXqHZ1QRhIwLb#pOx5>n>Du{o51G9b zT~>Ux7(V+}cWqN7|FxhZ6p8 zECYb`dV+;#(IjQ+m5J_JjD>*+*tgI+EF(junl46;B#T}tGUKFK@h$!gI1-I8r?7P0(Ze|D0d@)4v8R62%3)G1 zYE<0N$N&VId`=ofxx|Nx;6qD|!eT-KA{UJ8#50Mfid9>RN|_lEnM|jQGI{ll8t39D zR;i+1&`^=192xIXqkE?n;O86kIuU2@4a0DX1|F14#r^c;`Fe6DVwoFrJz$8vT$Ew^ z8@~qj*Gm-Nmb>k;O7T%>oI}o|Q+4E<^D@V!6LV~Sn@#v>9>ARGk3f4D*g}d~)zU(a z=*LIj`cuAusUWe?dJ_658~<#VPHnVOgjolPQmfRM8d)4m%uO*i@US)JC+y(h*sK_Q zPX@T;vJsFv^^)-G$-3wps!F@$pkaInv?%(#*Dc^EepBuG;aFTz{V?4L=fDj3z^Om) zm9HA8dO6Yr8drwCp*D3bv=cs2-+EV)ZHk<43vH64%}23*=wLr$!j@8pz)N~tN>6MT z|KCe4QlV2WR-rNL+Ag6EUapU@x;XHa*{jK19*2#FBs*`|Kpj^cQFPF|{s!F0H>)AH z_I+O9dP@$((Y3`6&ggO=-XglKG|^@;I~)NsS1ot|X43$kTROx!y<+YO67R$~eAs#K zn+fMCgXB!1x0w5oi@7i{*vF50Cohk`Hlj%DDfHZ=s@~X~Kc>`}+3)bLzXwb7};aNC44e@(+hDyo4?~r zhDg8`l^>L)-l8ua>$h*hW|zR}r{Isw6mG~on})qdfGF1kxDVm z4rRIr9xj(#|1#0gkP(%|cDjGF=qV^Dj&Tgtnzy*>CWGyW9LH(d6kKr(C8(Y;m~}Z= zU|BIR4sqb}mNu|IfM{8g@wkB83nX)unbrU`Oy-}n=yB(rutBo+2ytMa%_$M7;c}_q z7G~jGYJ825wld0gA75M36@}redOsKg;M}&Vm+9^Lm!@m_%dF13{L*t*@F>!-6RvU7b$_ZtDU8BN5!0vVY7H2vmuSTpvF|C>Qa;OF;(MaMn7kL@7+ zCX)I6GHyXX^QPW)%U>SRSb*3tZ*wnfMg73$p2DaeRW4z<2;s5tN%2r;ELpt%?zqVZ zvk&aJo4BBTMBQIo{fZDSP&4J20_ozHL|JFpAO;xHzt!%x5vr(uJseZdf*+6Xjr|y* z+l6qi_Pi4PHkAV;Zm0*SGpCdllLvf1VV#tz=cY5$7%(sjwCITx?cdZ&qf-{t60MUs z5kfsQ1O&7JKxNQ}pzDsrblMlh+;vPUH1ANTvrNh|DQH5U5iRBj^Q>Xm0Jysz?ppeV zm16Iq1H-kq#qZ8Y{yqRWpIdZyONx$uhgQ}i8EO?%Ivm!f4xDKeia#pvp z945KWm}S6)XSPNwnoJXJx$e8TQX^7*>AARSYvgOfAey-phdo)CeaOx#EeB@lR$s}B zw1VHDoY8YyEwkWsQP0_3CjhLGSKSh&dw|Zw-4%lKSqz{|KR)5#!t4{f?T2p86*B)u zX6+_a(@JZBb)eB*{e{jMa=O$v4Fi2^lxqm z^2TVVv(RH+pg9?=22$L3XPOX2Ixtp2gQ7Q-kdvDv2IkF73dUw?wK^1Cvq z)5&S}ft3@wvVBF;X~pt>#S&`OGWD)=_ypsn)dck0C4d};iibaT7U=sQ0?{@6_%#mk z*RL1S|3r8Z{?CM$#D6Oz|BdKkiR!i;phpg_&4(Xa1UorM-|;U9nM6@6l{|!?eUbs+ z7)r^`1waY^do~MpBBX+@!ZV1`bANyT{s{AHh91ejTbs9`aifq|!N{1_g#b-)eN83G z#w4?;C4}&KD9GQ?y)W(z=v3+C4F4XGV^*+(v3jm}>C?#QgoLSb<#F}9=Om5jWGwUr z^G88Sr)Kra0YsqR)0qAD2T22SYv@(X7aAxAiXH!{Y*pi8ww@!t!QZ(JW3D*_>aeI{V$ZfgsqdafweX9{~iBn^!~RS zW)eRk3B-UL@~z(79MfE#8r`WdYq_qmXxP0Y3J)S8PwtFiHg92lB*<6|Rlh^jP96Y- z*B6R7qbwo}OMIN=eGEl>jaO^;%e8*|h8a_*7575x>Ph5e6|7~wxv~Wdq+XdjUO^p- zP)=SCilhy}d1b%k=qYHOEwFC|Os6KQY^`1o%3V9L+-Mugi>6kf%Jy^84NOwuhk7TV zV$pmw?VfAGbJB-?%{0%`aiC1rUy1sMiI@C2aCEeOcG|-nv1W1P**9S;cbHk|HU_S} z(EWrYDE)`I{88Ta6vHk(Gh)MLNNEyf_1pTTM#Y~31!0rfKn2Ihz5m; zxfwDls<}?SB@70nzC-7Nhk%1wl%q@3p_zVwDQU#yKSr7ZwA${PKZNe|ez22bDc;YLj; zB`Stk%1lL7rDwO+qyuzF9@SxGkvWTx#TZkKRVQG#G7d^r>`W8fQZ-v$Ot5wB@!gU2 z=d{NckJ$5ZYv%JKNA0yV5spt%_*(JWq~mm_|MGmh(;FXg(})x_H8e#mGzBbGERm?N z-klgpiZnwsA&_)L$#f&CNJbh~Y+h#0fT@El8%Kf;4n}0pf~n2h*=<4GxQT+)Oq<@k zMvJcv)Jo|b>OHjKmL2pdqnufGxbN+#dCF>;_UF9Gw6TTE$&!MjBlQhq%_Qkd+tus{ z=#i-upYlH^W$Biu(nU=1?i=`AN>h*V-SH{z#Pyvf>wS-=%tmpEV;FK*TQ((WsBS7N z+r?yT^zERD?})d?#&M@rl2BZ1N5}Q9%jU#P8!57xL_iK49v;OoYDu;IB_c;(=G>Vv zmrEN>B88@$n>Z4}%r;0?H_nwRy2b-E}q^$T-tmCa%os+He(WH<%*%s9f zwit7hUt5GG(e@rva!qC(!B;k7xfU|ME4!K!R6Wc=vYDn%sv@B)jP6bqX3^|U#Ygt* zMC3DA%lyY|Vxmk(I_xnQHagyF<|Q%KcZ%^@Jj&uT>X~B@2mqd^%PV_5IU{vBQ>T$A zr_CyOO{VkQ7?x6EY^90`-(&+dd75OA)lj7DqKf%77+H>rp{PRy_30VG@ott}vd?6& zf^MNs30L8yGI_G?18Ged4NqkTvQyVAR8X1~z!n0OKo)bNhrvW%Gwr8LH1YiV8G}Jp zJ5L&v$jMLSGj&TtSr-mx!%ye|3^OTdcx=f1=nj@eHXViGl#b8(^yj6)5XYqPGo+0P z>xJK}P z(~9iD8Nh&0ct_%;>0kn;iBnIc?nSAIuI*VDqQ>iM6$y3;S>VO)Y5bf=3$ zDi|MnfWml@lcs3$Wxom=3kyn!0xE)}t`M;er!Oa!t6qf{1(g;g0HILBMUhA{km8|n z-BCna*nlv-?_2t-J=j`Uar)U*MiN7vl2|Zh$P66vR;mx@q>C<sLT!M1{%^%Bu#W2DLA?S-jnwoxuJ zob4B+rkFmn{xK{$C*sZ2AcoM;@!#5JUJRT zTtGvAK$AW@a}E50o5Uds1B&UWTp!-ylg7K91v@Q-=XZ6cPw>Zvk#=#7Nc7I)C75q- zwy#RLh}l(k)HCJ}_$;-TWzLb#)O~9*ZikyUv!BjGrL`4Aw|$=1n?F8?8{C_yo8>4@ zNb=XdKsLgfeG1N@}PGE!YqR(Y$`QOSjoebdU^ z$m+=>=3iO-YV(NkEZWd%ASYOmRl3Q#-IQ~vQ~U%bgZ>{w$@R|ENa)kXq6_&c-qhA9 zb!nmVB=DPr2on*a84r6w=42eeI>Yut?Jw~CT%pt!Fk$h)4HjW!h+zQ|by_{l=;E0r zs*?CA5|K^MBl^l={$d84LSu{Kz)!vc!7olOAm;L8b-evqKa7>r4& zG%Q4o`zVDJKI{673e738BaluHk0LfC{bn|Aqlm|0P+Gg-e{?(PZ20n8przC@oQ2yt zD;w=dq;OK&!0s;(gPwsP^-B4|@TV}inmHho4W=AM{u1)kP5Gv37`3>t223{s?fw;k zpx_x7-~DinD4|<~2M{}@jwC+08XqCvj8?xB9W;&yHgV{a;GZ-OcU10!^KSOZ?46vlp>-VSQIxf4wg>)zTl+(Dv)=b z`amUN*ozvhs%wPN=jVCi@&+?sKK*4<)U@cLlJXVhaZmGBtv>1}P@hc8eBwrb=)(^? zySknL?I-6K7d!KZ_JC6kd19z|KIL1D-`YpDQt0){dW-hF6wB2lNXw$MzwDv=upYwk z&&1kf)5XSCd;bJ$OQ%LX5Iz>++0U7n_6chiDV1RLi~490O0H;UZ65@nY8posNO&^= z?G34BOb(HeRy)Wmu&Zzx@2xOX^1jZV9<+Z6%Wk<#`tT}KVRB-BB~QZ^J-X7-B}wbe z_-e0S>a8elp>i&^3A)(wFMzN(RA`ArD5$85Dqa1No-E_`h%B37U>>3!Ha3?Xe zAzfWwpIcqcEWN;%Lazk)+sRhr;=m(qd>k_+bFQ;tF45&fO=Mue|$|GA@+-8qD#m@1TWKT)cn`^<;QcAI4;zFqc)OO#MjZR;= zas*|)Ri~%iDD>y&mhmYh>8Y2-ywjT`+2oq^E2-Bncly z^h)%aOre#rZ%q$}7HLw4?!mlbQpD8vEZLB~d#^KM=V$}jfgTl5SOvihf<|E*SaxK9MU0 zL~bDeRutdKkcElv;_jPk7lwXdmIC8UPsQry9x@;n-lEY!pjg|&j=gfYeUv_SC2sr` z@dhHCsjshWN44}0@Voza*O3PM&ja4)6oZ0+o#E5O8XDbeNdGsP8exL?RamroEmEo% z$FK$5b{_`VnBUT6>RMQ?PnUeGpy7Ic_d(dBeVGArEXq}?GB!>OF`_AVHOabx3$(8t zfOAd@@QqJw|FzI<+S>6R-~A1MN9v8ZVton&4OMLlRW_W&H;3OZEsUjde`D%U$zM4Z zxsyfeHU?@_nWI}>EW@-hE%NQ;eZ>5a7<^buOD+3j39)pBp}|{ z2y8fPQwkKabg74}cVs*t5MRiX0Pr!19~SWGbhKR`8pzGwao>ozRF>kVOo}pXFLlV7 zC=Ci4^<%AtE5^tzGyogXmcSx9#3}LPlF~&leK=v}CK`6)4opD{*TPk)zsa{1PE{qE zQ)DhA@Dtl#ax9wN%b9n20cA+;f%8PF3DwE;O_YfhlzU*9cqf$0uq|Jt$l1`B>ct8@ zD311YJtMvp0CztR?O@i&!Cx_j^$3Npfr-TM9S!D(;eKsTyca*k|?b6Y`1 zr1R+%zAi#ZeQp%-!NpukELW2_yOG1^&Q1?qOoRNA#NtuB<3xz#%S(4$1z6VZR) zWH~;#jg)ln^M}0{56X?W`=d(Pf$D{$ucbv=G-sz@6pFnbmpKCn#HC8I^F^;pUAQQ( z^@w=ypts-K$bu@Nr%Vd!%Z(1iJY9WCs(`QB-5*WY7X#h@?S*W`xbW9?#LT{uU9xhNul6+ z4ylUN zeg3Ll*yYjJNPhcjU4@Zm{2bmHKo+kdKq86!6eUUH5% z;tYcI8~OmpEjc}%f;WoWqi{3|%820I2bDG?U+y$v_VJ@juTV5dgK%QQ=iub1j<9@|6bV}1kv`DR6hW@AxGH_R&8g;`QqPVS zLy4eMh*4%ZI|X6_sc09qDkZ#41;k9f@t|DnE~UKCDB7;*ZrR4#nk{#@ta!b7RgvQo zh8SDLNv)*oOtpLXE(2-bS7YWHuq zChacV4;|i-b19-gDay+07AU*08=)rEoMz9UM4PU%zJrU9^ByS|BTJdJ#GR1H(RArR zT6Z5MZV|~+@bf{T8V>1z8cH6t78?+T!sdG<-7TOf5m7*4UlgQW#OfGRBTtRMQ5YXs z3ylbh1^c}M3glR!7r#ly#!Qfak1a6@gl@rnGQOjz(OFKxjd+w9EckGCm;M4WCwVI%HOZu9856^>RNR+x<{DN7fdn)! zyO+ItWnX?2pRbp|cTIji)_f=%8#EQukLaGCjUU_uNh-&tJTVt-RHDiN)jGa1#jA*( z6o6})GUli)4>6HV(;dc5&#fG~kv>?p(R|x(CkrzY1uL&*uS9rxDpVY5KjdA#Z^gB% z(kvnhNb)sWOo~iZTRuv5HMlKmq3-gjum^a&YWHve^G{QN&ElP;I{{YCJq0;U_QC1= zp|uhBeCp;?@?wIuh+O3w23K6CyapDjJMwk?PteYHrE%PW=} z{eu)JF+wa61|2r~wb7bRb=X-$H}nmWw+~V;pSTk#RRkdyM_^sBSq&geW~03{>?4oD zV||Xx!wy%#E2kj^ep-`nK3BoUZwxhduPSTcn%2$CfM|3B$i@i}&ZA0oZvqRd&nag$Row}Z3 znrnObbTaJ?E`+wb$PdL=#kINI_CDQM>`=djl<~c4k)`f38xUY&z9HIAp4l5&3&o`m zR++1w`y&3GltE>PcFkxn?J>!O`@s?YJyCuegyg6GLD3G0&I!L%p|CrWZPE-tC=+Ek}kmaSV;ypJmXA6&fBoMRmBs z+@f(Z^d8kPeE_Z%lmyAN_>XJcxpg1KYni5((`8JcwNp=J>|u!q>UQ=C zdR1Z>*Y-iEb}&WJN@u2h;c*=jb)&r8j>SHKtjW4fmXGdDQG*+p_0TAE9?#kXi*SFk20jj@E>g3@KP4`)kA4wz0u)PJZTO8T*qoYP>&Y|g^0SV{HEpHX7FC{g% zT%XDwcI=y>K6CD-B*CZdS8e*}nf^$Kos2XTVPW@S)6sn%&M@TRee^siXxFSZ;+ktC zl+G1#<3j7mIud$x)M%^or@aR^g!3=DU0=>bT6nR;9*JPOp6e`}K1kKGuCJ=7sBTU+ z@bBXU@y)Tc==nshwQkEE-&E-dJ9@YoR@8+ z4Ppp_4Q+W>)d0^dXlZaCF%yG5CV!zIzi(^i2sNs9BZ5b5G&q zzVyN_ViL%XoqLb<#5)3B>f$|>dY9zvprkyS)Z@`Wcu40evAPz72sLS7OR^VXX9v5P z#kr8xx&M{J>uBGLSVQE*OC%#@o^xyN8f&g5T|XE5YC*lcm0jV_1`i8(Z$BtQ`9Vij z0Z{~b;pg*VERejW(Iwzm#}>30H?$+zYhmia1{EZA0+DP@VX>E33mtjQ(ESwh4Uw1! zN)o4(qb%W2CHXa8{74lbg&owx1D4)(Gjr(MRO~|H!#sHHH-Nm)TcR_Z;&Up8f@Fcb zZ%8Kwann~yiBV43gJK4ko^x?&A|rsVP`uzDWKbn>uDCsEs=Z%Ck~DfHVUMH_OPy!E zP`X@sjcq<*oy{q(o6dlwj6UN|7T29L$F$gKP&Z*VKCkU~R;+qC1V4lBFe9V_x~f9N z4hlEiq`VdGq$LRvUT%%;UUNOUCwVe<81I9_LbgEv!9Qf*huZv7ygTNSVd#?Kp1(5y zl2neJowGcPe0w^?NkGublY1|9g~HOvUC_H0Q^8;Q=>7Ko5o26*pWHlSr^)7Wz05wt zt)Tc!s@Nn<q~^B_&I^ha|s$Bh-xFc84FI%B26gnErYyn=OnpN=N#Lo{*weM|{z^XL||%xp)ebaU)aTI#SU)5l}# z=kKnLyi*=ozi3QWRgZ{562J1VlyEm*1fYkPtNIL7Fm$Yj+GdY{yh+~;zRAn|qD!Nc zLTd0v^=8l@L6wFvTIMjO?H-gaG?ZHaHYI&^3lJ^$ts9-4SYh)Tn0}ef6qA#O4gF9nK#?RAS%>siYM`ywu9K0< zp{Qd_bG7KueJ|(Zbx0|aO1YYn`l{!Sx?Ey*ogM5Ta_ zq*&hB!;efrVT!%ISp{pM$c*-`R4;Lo?(6%z@sSfRcI~@+sh|(ogV0rN2-PA4Z3_;) z1mPZ!gx`0TlN-&GP!{A_+8qzaT!)aAg_l?0BmA~~PK`cuQSB86=uaIsB9%!}A1Vc6 z=tsIWNbBGLoM*ebz1wZ5)et4n{b*eV8BGU|U^CvS^u}-1BaGChPbm~JeJOYvm6i#T zN(O3scr*g~P@} zTEwv4f+8}%8=tUdTS3H*Q4V#koD!P=t_u4x=*`0I=S+1X!LidJ>g%pTesoB?jZjV+ zflU!BZ^*n-=GhBM`U@V~MP?RBP64TD6e=)zgOd0@D+WZ_p(bt(mqqDule17G%F`f- zb|TT`iJs4Df~taED&%}J4`Np3pm7C^>PozA!-#y-PZBXCMvYp5k! zKHWZf+^AFKx<6an5E`>_g6k_?HNW)rFeWax5z=}7M*fV$2jzPFjB&F_YoxX-4eh1)VI(l=kBfZamb!D z{Wa&Pu{& zbW+viAKnvYT;UsVR5K3jp3@^#60~AVhY5VBINdL})kALb%rg_N8&n8uwpO;L&njf7 z&e-!+b-Z(i+eE7~75eR$9lUc-Eeq-0qzvOTJ)e=`QAw@R`cZqQFp&Mv0q%IAH`*)8Eko-E8Dr0N*pWStX<9FpSgfK(q z{ac^YCOnVO7qVBeeqi!Kke=f)@a28U>o5-7bIw`R?pB{N5?mw7gQxXK9GV!uZqT}M z7j14p#^)7@89SyY8AtW&GJQQ&dU}ob)Z38S((>}C->O+=Y+troydx-V%wsC3wekdh z79MsEubf7IbJ_Zc551zgJ21DA9(+w&4&!VTs^zb)_$<%dvD92DW*Wu5Xhi+3dXrPn zSCJ&q)|`D92dbQk`o{X&l1ActMq%SKo#UUqhaVxcmDrkx^b;fv=$_3S1(mAkw4pZ! z;I^F+3#>pO?82VP^B5lQ^kd1qbLS{pE%s}vFp*i22}OX2coR?C7Knyf+v#nVpawqvjRlipV)kI z0(D@LJff=Bky{`HhO!3Zg>*4NBb6hEQ^R#j>Z5LQncDkHtrhP=AonG2%e${rxx#|- zA{-j0Y>rIecNtc)$k%he)W609fQ>mhJs;QmJRPB04Uln@_fE5_y$)9d2`IjPd*#tr<|?ne$)N%$&l>S~tzOtVh)&24g%jg}sp# zDfInFUnO>Ufj;E-qkw@Tk{UePyIeCOk?<^&F1U%PA01^^ZVu1nMqY~hc2msPg7o zwXrN7z4+*mW4tPFgsu5r1@^_K;A|Qsru;4&{0yg78O?_;_T#j}7FrSZb3RRnM>s6@ zHJCLz>aSg{A3bP(iVdf$r^`ejuPYsLIucb6#HnmZ#o1z)PUyjyJys%mxE+EiDeW~G zH-^TGIXA6q*mLw@oG}$wE&5Dl99Kj+x2#2v=U_K$(ce3#o)Qp@wa;^H-!5AWvmf{PxKVreFV@K)SI(8a6vwm^j{b@Rl$D z82{%l!9t$pvP<q*| z(Z0^>eeoipH^0EI3NI^2)(xjJs?~8uR>BCed0Xfb7eBlDc$`m8U-U@PBY4{?Q}5VN ziK-Xu_GB$VNu^g3?@7mJ@}^i?x!BUQ$r$Eypk#>io<~9=a>_y1!OO0R7%~=k4ztUW zK3n|9W?|C+ZXFj;Zr3nD$r{&;?7WLj?fpQ{Av4vFh)!5eEKa4C+nQ4&?;SkKK74V` z$&Syd>JbHWW7JQD8FLZz!_S3eBv~*;?KNJsvJ4-kI=&em_;Pt zP<+^KA95qMQ(=>;oPxSWyJ3oWP)JcpL($fC)C^)?pQqG0oCCzCLvjn#}xs_wvo|g?Ntuige(3spM;T? z>0U{&Ie`R`<|t56T6AP5nA!@GCTgDXQYAN^v&?l+3ilgmx4^*%kJL7Eio)UyVu`^k zi3EzmFZl>q#k^h~;R?1o65uWg*sysjkRCGFw>$J!Y z)ljpmxzUHbi}hOyb(DJ9Ybf`*<1A>Aqo3K$lM&pp9=$^r;l9}*sGT-L*Zqkyr3MkO ztaP6N^lm5QVW+ZV6(&QxnTAhV+?fG+^vW8ls6}x9vOBYePh&^f$?f;@r^X^O-V-PKI^TSxDF+S350;cZmjXSiBZbUTNxPj zLq-E-cO`ub-_}(1#ytYl7bOHsE^h74Hk`FD7FCs_F z&?=bN_*&Zrn&pyF|aSub&v_bPom|X`{_O6NI$#(g$*n)+ak!HHQT<; zi~FTNhU)00jfj<7QHZMxOKKAc#lh#4n_oX!o*#j6rXx8wv6XIk! zDdZaBhw!_V&VPC2@9!F5VOxNyg_*O1!7r++brfxEhZHb>ey5$GPFC1WMfHHXfLAbkNnrnh z&=LWM(O%*z?H9@&*F#}3(xw{+Bk1yl?eL>twlsYEs~8?d2Rn5o23bNH550S*_evkR zAZb}zr=Fd$MZwNkZ(ACm{+u_?iw|4TlVma4=(MMinYxGy-K>-Fz^FuHX3r{WFK(a# zb`2M^&FPti%k_t7GMy~t$=2bi-UhP~jFMbS~1)dv_swc+;yR zaRvm%=}>iV=Kkbt>})^w6KJFZsG>3`w2wQkenXu4OW1iwgoc0dEyi%`m>xk^FVMdw zF4O$_g7|mTiMp8>Isc+Jsj7{&(m0+EYNb&}1Dqj0iU~(Z>?e`~^VCQ87%gbtYe;6} zZXj!XB)FWY`E#g zKgk=qkx8vUvoPte87D(cwkTh5oImAT_s%-XJoWBB--y@qfm+b5r{3SGaSXl8jEvr% zl+@@Gz08~RVt?41`ZoN z&&%Jay$UHWW21HjuX==mF0jYetv-`u<-b*Vt3r2C>sjaNktdHp3L9YY(bQzsCyxd$ z7pNkE#qdlmp@3GpRQXM!M-S0yM%-koji!$qoNDHX{B}XEF4G&-=&S^d=@{(PLiNXS zm1ITttkAfUQ*AJnLIE4D_Zjp=YUEzmNwmq(ri3NE*XSdF^Dlv^TK_oSGQvQuY&i(X zvz|ahbaG+>ji-x`vqH<8jkBV3mpwrgn5sU!1+R4!lB(v(28QrzEw*W*Kvxu&9^(g8 z3wzt358%6>+2crlVG|oI;VFr+I>X#UHN+slw~e18yeiczH=Q&h?IQv=1(=HK_OL?N z8esZRJ9%@pfxeF|`;7X7kG*@KNrx1Up^y_f(}%+QpJLwlF>A3GDGEyk7Z7A&tH&)9 zGZ@ta4edwh;O%0XiQp{8XS1*-R9!cX5u@-{TT4o`RPgC@)E-vL>zl<>Rar*Qw@c~8 zlFifF<>%&Tm0Q=>&;Fk^a@YpH(y%p!E@=vX;9c2clD0#e~7?+G%>`kVe5Pwmh0u0Z;-Y!^`Vr< zvzDku2#)8IHwW=R)P<%9u~=>5xo40meq5mS6X*|2&wE{*k+YN{o^UeqyqU&;P9Lyq z&A8=ro#o2P9QeA~z^p?L#{x9yf!ka;3wj~>F+*b9O0`)&>JhsHPNJ<~4dAhM!Xx%{ zrx)=vq*YH%G~>l1U$I-cKv6-nzR7`ABBPVONUS3hV3^`5#<4oo*Uh+87|yGB@gO&R z_`BL%Z_wN$X*d#BBiMNsb>qnjOIMx@_FrUSY|0ZiGBls!InsCvpl zSt{+RX0=hww9oZ)pPLV`y9<9{eAW@h9ki7?9h;k9y0|Umg*C5_&L9bT55FG}f-@7= z(3wnkJSn6rhj4Um(AW5W(Zuyc>%nX*YwcGVRn?i2GaG-D2#ioGlQe`OmRll!1o56Q zjN{G(19_3htCJGqo*)WE=WXFNaxrejuMKgZBgkKw*>iR7LFt;2^ropwR;9T{VW7~R zx>(M6GA*KDb#n%~$*Q=tWk$=WuO;w9^0BUPk34%dS#fC7TtqlVfMAA?6@p(q@0H!o zb_0@2aZb@N16?OePY+v7(WIx5FF)vV=d$cnAw-rF79_U6T{RzQ*Ii#{Yi(nmhZ-dVG3&twm6NnLT_ObNI8D z?$%vkPYeUw$NUjCY!W^Cp6|@yc}o_T-55qF>e)6(GFxD-=iHHvpIkS7VmRqrU?VY{ z<$G$<+KA`%bcZMnPAKN&(lDiGW5fK(PFnlmfKk_}y*8diwCu8*dBW22gQevhW(C62 zkGU+iy=cKm ziKR>`jUIR~ZLiCmiAznM<9Ig(2*lq%c55G+bMrv8?Ooa}GH9*RQ-LZ?B|i8xob#2M zy9@Vff2$vFji&NtZ*_Boa~&I?qrtG*{b1K+<(bv+mQ6G}ra=c0>Z7%WD{}d(2ohFk5GQjh zzG|CYxM@?dHVc%Y3IIpv^VG;b1nQo!)`Toj36NwO9`dz~uBTY{v(9Fkl{a)cc?MrL zG&|LOQud7KIzd%k(g7}bA}&&^2WQ2NF@Jz}RF81KID^_gunIaLlk^UXIz!oOwUyj) z^+LFYc3qAMTacfK6krSYC5w+K!}1m7y$P{71~+3rm-PqDl4Ffj7kXRkoE0c}Q6Fxd0JzFy#jzttYd5r2)Z**Zu6(rTZo z#JUzXt2$RM*)J-lT7#v>@uYnyC*q|CF&7pP)ewcxhAv%~I$edQrHQ7{yi#F-c7NS| zA@la$-Q7NgLAu}|gt>6_FXjUDJxnM}C`3dE914mI>O)l?@g@osR3kMM6guRJ`J>5- zDhn}6%84<{ND7O}DT%5m!$CprLhlxPKf1B(@77F&d|m^2VgBHT1=6JH|J=w5$w`Wd zDk(F`iv3qtM4@h}cb#R_Ag{ZgJl_l*4GPNE!Hn7LU+tTijjS!0zr*1QIhZ-yKzfct zkX0sjzgSEDe8^N|&?OF#RYW>u6>*2#{bmv9ztj08FY;&SQSWJQB_Yn!AdBNWyx}*~ z#Q2@_zX`wx_!o0!@ zhJR1zPjwtWOAZQ?S`9$r$qaI8pZy@YiuVVS(f|nI;jh8{HCzbE!^z|yG_9x05;!3f z8UUHlABRax_6M4>7LJY(p4nfjkVyFxNw99gQWHdy9@3lp$6*dr{DI`(TCN5TCVxIm zz3W43S4iMc+V&iDsHDgdkhpYZ&i>iy%7 z2?xtRtNz2>eb2i6(Rl{Pe>nfwO#KEE{_~gldqnFUF87@w`o4YS9bMs@6)F8z>K{V=`&NT@+g{&n#qgJY|DYA+&&U3K`@J874c+878h>wm z{Pz}yKkIy7^?cXN@0;nH{+Z4{OxE`W!*|8g-|U6^F9rTvDfQ1s{(UX$k7vQvYMdv{Mz{+ '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 00000000..a51ec4f5 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,82 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem gradlew startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables, and ensure extensions are enabled +setlocal EnableExtensions + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +"%COMSPEC%" /c exit 1 + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +"%COMSPEC%" /c exit 1 + +:execute +@rem Setup the command line + + + +@rem Execute gradlew +@rem endlocal doesn't take effect until after the line is parsed and variables are expanded +@rem which allows us to clear the local environment before executing the java command +endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel + +:exitWithErrorLevel +@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts +"%COMSPEC%" /c exit %ERRORLEVEL% diff --git a/infra/README.md b/infra/README.md index 5a39e862..262afec2 100644 --- a/infra/README.md +++ b/infra/README.md @@ -5,3 +5,50 @@ В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. + +## Топология lab3 + +`nginx:80` — единственная опубликованная точка входа. Он динамически разрешает +Compose DNS-имя `api`, балансирует без affinity и пассивно исключает peer на 2 +секунды после ошибки. `api` не публикует порт на host; две реплики получают +общие PostgreSQL и Redis только через сеть `backend`. + +API readiness вызывается Docker каждые 2 секунды и проверяет Spring readiness, +соединение с PostgreSQL и наличие таблицы миграции V3. При SIGTERM Docker даёт +5 секунд action-script или 12 секунд обычной остановки; Spring прекращает +приём новых запросов и завершает уже принятые в пределах 10 секунд. Nginx имеет +connect timeout 500 ms, read/send timeout 4 s, две upstream-попытки и обновляет +Docker DNS с TTL 2 s. Максимальное обнаружение зависшего backend — 4 s; +максимум двух последовательных timeout-попыток — около 8.5 s. + +Flyway разрешён только в one-shot service `migrate`. У API задано +`FLYWAY_ENABLED=false`, поэтому запуск нескольких реплик не создаёт гонку +инициализации схемы. + +Restart hook поднимает зависимости по фазам и не запускает Nginx, пока обе +API-реплики не стали healthy. Это не позволяет Docker DNS объявить +прогревающуюся реплику через уже доступную публичную точку входа. + +API ограничен 0.75 CPU и 768 MiB на контейнер, Nginx — 0.25 CPU и 128 MiB. +Порт публикует только Nginx. PostgreSQL и Redis имеют собственные volumes; +API и Nginx постоянных volumes не имеют. `container_name` не используется, +поэтому один Compose-service `api` масштабируется параметром `--scale`. + +V3 — additive-миграция: она создаёт новую таблицу и индекс, не изменяя таблицы +lab2. Поэтому старая версия может продолжать работать во время поочерёдной +замены, но новая версия не станет ready до применения V3. Автоматический +downgrade схемы не реализован; rollback приложения оставляет неиспользуемую +таблицу V3. + +## Swarm node-failover topology + +`swarm/stack.state.yaml` и `swarm/stack.app.yaml` описывают дополнительный +трёхузловой стенд. API имеет две задачи, `max_replicas_per_node=1` и доступный +третий узел для переноса. Nginx использует `tasks.hla-app_api` через overlay DNS. + +State и LB намеренно закреплены на единственном manager, чтобы граница опыта +была явной. Защищаемая зависимость — процесс API на worker: при потере worker +Swarm переносит задачу, а сессии/сообщения остаются в PostgreSQL на живом +manager. PostgreSQL не реплицирован, его volume локален; потеря DB/manager не +переживается. Также остаются SPOF: Nginx, manager quorum, локальные volumes, +Docker underlay и физический host, на котором запущена учебная DinD-модель. diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 00000000..04ed5162 --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,70 @@ +worker_processes auto; +worker_shutdown_timeout 10s; + +events { + worker_connections 1024; +} + +http { + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log warn; + + # Docker embedded DNS returns current Compose replicas. Nginx OSS 1.27.3+ + # refreshes these addresses without a configuration reload. + resolver 127.0.0.11 valid=2s ipv6=off; + resolver_timeout 1s; + + upstream api_backend { + zone api_backend 64k; + least_conn; + server api:8080 resolve max_fails=1 fail_timeout=2s; + keepalive 32; + } + + server { + listen 80; + server_name _; + + location = /nginx-health { + access_log off; + default_type application/json; + return 200 '{"status":"ok"}'; + } + + # Message POST is the only non-idempotent HTTP method that is safe to + # replay here: the API requires Idempotency-Key and commits the message + # together with its shared PostgreSQL idempotency record. + location ~ ^/api/v1/chats/[0-9a-fA-F-]+/messages$ { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_request_buffering on; + + proxy_connect_timeout 500ms; + proxy_send_timeout 4s; + proxy_read_timeout 4s; + proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent; + proxy_next_upstream_tries 2; + } + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 500ms; + proxy_send_timeout 4s; + proxy_read_timeout 4s; + + # No sticky sessions. Failed/unready peers are retried elsewhere. + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + } + } +} diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..8e4e21b5 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,8 @@ +@echo off +setlocal EnableExtensions +cd /d "%~dp0\.." + +docker compose --env-file .env stop nginx api redis postgres || exit /b 1 +docker compose --env-file .env up -d --wait --wait-timeout 90 postgres redis || exit /b 1 +docker compose --env-file .env up -d --wait --wait-timeout 90 --no-deps --scale api=2 api || exit /b 1 +docker compose --env-file .env up -d --no-deps nginx || exit /b 1 diff --git a/scripts/restart.example.sh b/scripts/restart.example.sh old mode 100755 new mode 100644 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..9d3a8cd0 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +PROJECT_DIR=$(dirname -- "$SCRIPT_DIR") +cd "$PROJECT_DIR" + +# Preserve named volumes and the network. The migration job is intentionally +# not re-run during a same-version restart. +docker compose --env-file .env stop nginx api redis postgres +docker compose --env-file .env up -d --wait --wait-timeout 90 postgres redis +docker compose --env-file .env up -d --wait --wait-timeout 90 --no-deps --scale api=2 api +docker compose --env-file .env up -d --no-deps nginx diff --git a/scripts/restore-api.cmd b/scripts/restore-api.cmd new file mode 100644 index 00000000..2fc83566 --- /dev/null +++ b/scripts/restore-api.cmd @@ -0,0 +1,6 @@ +@echo off +setlocal EnableExtensions +cd /d "%~dp0\.." + +docker compose --env-file .env up -d --wait --wait-timeout 90 --no-deps --scale api=2 api +exit /b %ERRORLEVEL% diff --git a/scripts/restore-api.sh b/scripts/restore-api.sh new file mode 100644 index 00000000..6f90f154 --- /dev/null +++ b/scripts/restore-api.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +PROJECT_DIR=$(dirname -- "$SCRIPT_DIR") +cd "$PROJECT_DIR" + +docker compose --env-file .env up -d --wait --wait-timeout 90 \ + --no-deps --scale api=2 api diff --git a/scripts/rolling-update.cmd b/scripts/rolling-update.cmd new file mode 100644 index 00000000..a881740c --- /dev/null +++ b/scripts/rolling-update.cmd @@ -0,0 +1,16 @@ +@echo off +setlocal EnableExtensions +cd /d "%~dp0\.." + +set /a REPLICA_COUNT=0 +for /f "usebackq delims=" %%C in (`docker compose --env-file .env ps -q api`) do set /a REPLICA_COUNT+=1 +if %REPLICA_COUNT% LSS 2 exit /b 2 + +for /f "usebackq delims=" %%C in (`docker compose --env-file .env ps -q api`) do call :replace_one %%C +exit /b 0 + +:replace_one +docker stop --time 10 %1 >nul || exit /b 1 +docker rm %1 >nul || exit /b 1 +docker compose --env-file .env up -d --wait --wait-timeout 90 --no-deps --scale api=2 api >nul || exit /b 1 +exit /b 0 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100644 index 00000000..17a6c19c --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +PROJECT_DIR=$(dirname -- "$SCRIPT_DIR") +cd "$PROJECT_DIR" + +original_ids=$(docker compose --env-file .env ps -q api) +count=$(printf '%s\n' "$original_ids" | sed '/^$/d' | wc -l | tr -d ' ') +if [ "$count" -lt 2 ]; then + printf '%s\n' 'Rolling update requires at least two API replicas' >&2 + exit 2 +fi + +for container_id in $original_ids; do + # Replace exactly one original replica and wait for the replacement before + # touching the next one. Named volumes and state stores are not changed. + docker stop --time 10 "$container_id" >/dev/null + docker rm "$container_id" >/dev/null + docker compose --env-file .env up -d --wait --wait-timeout 90 \ + --no-deps --scale api=2 api >/dev/null +done diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..4c96d747 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,25 @@ +@echo off +setlocal EnableExtensions EnableDelayedExpansion + +if "%TARGET_INSTANCE_ID%"=="" exit /b 2 +cd /d "%~dp0\.." + +set "TARGET_CONTAINER=" +set /a MATCHES=0 +for /f "usebackq delims=" %%C in (`docker compose --env-file .env ps -q api`) do ( + set "CONTAINER_ID=%%C" + set "SHORT_ID=!CONTAINER_ID:~0,12!" + set "HOSTNAME_VALUE=" + for /f "usebackq delims=" %%H in (`docker inspect --format "{{.Config.Hostname}}" %%C`) do set "HOSTNAME_VALUE=%%H" + if "!HOSTNAME_VALUE!"=="%TARGET_INSTANCE_ID%" ( + set "TARGET_CONTAINER=%%C" + set /a MATCHES+=1 + ) else if "!SHORT_ID!"=="%TARGET_INSTANCE_ID%" ( + set "TARGET_CONTAINER=%%C" + set /a MATCHES+=1 + ) +) + +if not "!MATCHES!"=="1" exit /b 2 +docker stop --time 5 "!TARGET_CONTAINER!" >nul +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.example.sh b/scripts/stop-one.example.sh old mode 100755 new mode 100644 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..3c6cc82c --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu + +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +PROJECT_DIR=$(dirname -- "$SCRIPT_DIR") +cd "$PROJECT_DIR" + +target="" +matches=0 +for container_id in $(docker compose --env-file .env ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id") + short_id=$(printf '%s' "$container_id" | cut -c1-12) + if [ "$TARGET_INSTANCE_ID" = "$hostname" ] || [ "$TARGET_INSTANCE_ID" = "$short_id" ]; then + target=$container_id + matches=$((matches + 1)) + fi +done + +if [ "$matches" -ne 1 ]; then + printf '%s\n' "Expected one API replica for X-Instance-Id, found $matches" >&2 + exit 2 +fi + +# Five seconds keeps the operator hook below the test's ten-second budget while +# still delivering SIGTERM and allowing Spring's graceful shutdown phase. +docker stop --time 5 "$target" >/dev/null diff --git a/scripts/swarm-lab-down.ps1 b/scripts/swarm-lab-down.ps1 new file mode 100644 index 00000000..e894742b --- /dev/null +++ b/scripts/swarm-lab-down.ps1 @@ -0,0 +1,19 @@ +$ErrorActionPreference = 'Continue' +if (Test-Path variable:PSNativeCommandUseErrorActionPreference) { + $PSNativeCommandUseErrorActionPreference = $false +} + +$nodes = @('hla-swarm-manager', 'hla-swarm-worker1', 'hla-swarm-worker2', 'hla-swarm-registry') +foreach ($node in $nodes) { + $existing = (& docker ps --all --filter "name=^/${node}$" --format '{{.Names}}') + if ($existing -eq $node) { + docker rm --force $node | Out-Null + if ($LASTEXITCODE -ne 0) { throw "Cannot remove $node" } + } +} +$existingNetwork = (& docker network ls --filter 'name=^hla-swarm-underlay$' --format '{{.Name}}') +if ($existingNetwork -eq 'hla-swarm-underlay') { + docker network rm hla-swarm-underlay | Out-Null + if ($LASTEXITCODE -ne 0) { throw 'Cannot remove hla-swarm-underlay' } +} +Write-Output 'Removed the disposable three-node Swarm lab and its nested volumes.' diff --git a/scripts/swarm-lab-up.ps1 b/scripts/swarm-lab-up.ps1 new file mode 100644 index 00000000..412af524 --- /dev/null +++ b/scripts/swarm-lab-up.ps1 @@ -0,0 +1,138 @@ +$ErrorActionPreference = 'Continue' +if (Test-Path variable:PSNativeCommandUseErrorActionPreference) { + $PSNativeCommandUseErrorActionPreference = $false +} + +$projectDir = Split-Path -Parent $PSScriptRoot +Set-Location $projectDir + +$manager = 'hla-swarm-manager' +$nodes = @($manager, 'hla-swarm-worker1', 'hla-swarm-worker2') +$network = 'hla-swarm-underlay' +$registry = 'hla-swarm-registry' + +function Invoke-Checked { + param([Parameter(ValueFromRemainingArguments = $true)][string[]]$Command) + $executable = $Command[0] + $arguments = if ($Command.Length -gt 1) { + $Command[1..($Command.Length - 1)] + } else { + @() + } + & $executable @arguments + if ($LASTEXITCODE -ne 0) { + throw "Command failed: $($Command -join ' ')" + } +} + +function Wait-InnerDocker([string]$Node) { + $deadline = (Get-Date).AddSeconds(90) + do { + & docker exec $Node docker info *> $null + if ($LASTEXITCODE -eq 0) { return } + Start-Sleep -Seconds 2 + } while ((Get-Date) -lt $deadline) + throw "Docker daemon did not start in $Node" +} + +function Wait-Service([string]$Service, [string]$Expected, [int]$Seconds = 180) { + $deadline = (Get-Date).AddSeconds($Seconds) + do { + $replicas = (& docker exec $manager docker service ls ` + --filter "name=$Service" --format '{{.Replicas}}' 2>$null | Select-Object -First 1) + if ($replicas -like "$Expected*") { return } + Start-Sleep -Seconds 2 + } while ((Get-Date) -lt $deadline) + throw "Service $Service did not reach $Expected replicas" +} + +foreach ($node in $nodes) { + $existing = (& docker ps --all --filter "name=^/${node}$" --format '{{.Names}}') + if ($existing -eq $node) { + throw "Container $node already exists. Run scripts/swarm-lab-down.ps1 first." + } +} +$existingRegistry = (& docker ps --all --filter "name=^/${registry}$" --format '{{.Names}}') +if ($existingRegistry -eq $registry) { + throw "Container $registry already exists. Run scripts/swarm-lab-down.ps1 first." +} + +$envValues = @{} +Get-Content -LiteralPath '.env' | ForEach-Object { + if ($_ -match '^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$') { + $envValues[$matches[1]] = $matches[2].Trim('"', "'") + } +} +$password = $envValues['DATABASE_PASSWORD'] +if ([string]::IsNullOrWhiteSpace($password)) { throw 'DATABASE_PASSWORD is missing from .env' } +$database = if ($envValues['DATABASE_NAME']) { $envValues['DATABASE_NAME'] } else { 'messenger' } +$user = if ($envValues['DATABASE_USER']) { $envValues['DATABASE_USER'] } else { 'messenger' } + +$existingNetwork = (& docker network ls --filter "name=^${network}$" --format '{{.Name}}') +if ($existingNetwork -ne $network) { Invoke-Checked docker network create $network } + +Invoke-Checked docker run --detach --name $registry --hostname $registry ` + --network $network --publish 127.0.0.1:15000:5000 registry:2 + +Invoke-Checked docker run --privileged --detach --name $manager --hostname $manager ` + --network $network --publish 127.0.0.1:18080:8080 ` + --volume "${projectDir}:/project:ro" --env DOCKER_TLS_CERTDIR= docker:28-dind ` + --storage-driver=overlay2 --insecure-registry=${registry}:5000 +foreach ($worker in $nodes[1..2]) { + Invoke-Checked docker run --privileged --detach --name $worker --hostname $worker ` + --network $network --env DOCKER_TLS_CERTDIR= docker:28-dind ` + --storage-driver=overlay2 --insecure-registry=${registry}:5000 +} +foreach ($node in $nodes) { Wait-InnerDocker $node } + +$managerIp = (& docker inspect --format ` + '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $manager).Trim() +Invoke-Checked docker exec $manager docker swarm init --advertise-addr $managerIp +$token = (& docker exec $manager docker swarm join-token --quiet worker).Trim() +foreach ($worker in $nodes[1..2]) { + Invoke-Checked docker exec $worker docker swarm join --token $token "$managerIp`:2377" +} +Invoke-Checked docker exec $manager docker node update ` + --label-add hla.state=true --label-add hla.lb=true $manager + +Invoke-Checked docker compose --env-file .env build api +Invoke-Checked docker tag hla-lab3-api:local localhost:15000/hla-lab3-api:swarm +Invoke-Checked docker build --file swarm/nginx.Dockerfile ` + --tag localhost:15000/hla-lab3-nginx:swarm . +Invoke-Checked docker push localhost:15000/hla-lab3-api:swarm +Invoke-Checked docker push localhost:15000/hla-lab3-nginx:swarm + +Invoke-Checked docker exec --env "DATABASE_PASSWORD=$password" ` + --env "DATABASE_NAME=$database" --env "DATABASE_USER=$user" ` + $manager docker stack deploy --compose-file /project/swarm/stack.state.yaml hla-state +Wait-Service 'hla-state_postgres' '1/1' +Wait-Service 'hla-state_redis' '1/1' + +Invoke-Checked docker exec $manager docker service create --name hla-migrate ` + --mode replicated-job --replicas 1 --max-concurrent 1 ` + --network hla-state_backend --constraint node.labels.hla.state==true ` + --restart-condition on-failure --restart-delay 3s ` + --env "DATABASE_URL=jdbc:postgresql://hla-state_postgres:5432/$database" ` + --env "DATABASE_USER=$user" --env "DATABASE_PASSWORD=$password" ` + --env FLYWAY_ENABLED=true --env MIGRATE_ONLY=true ` + --env SPRING_MAIN_WEB_APPLICATION_TYPE=none ` + hla-swarm-registry:5000/hla-lab3-api:swarm +$deadline = (Get-Date).AddSeconds(180) +do { + $state = (& docker exec $manager docker service ps hla-migrate ` + --no-trunc --format '{{.CurrentState}}' | Select-Object -First 1) + if ($state -like 'Complete*') { break } + Start-Sleep -Seconds 2 +} while ((Get-Date) -lt $deadline) +if ($state -notlike 'Complete*') { throw 'Migration job did not complete' } +Invoke-Checked docker exec $manager docker service rm hla-migrate + +Invoke-Checked docker exec --env "DATABASE_PASSWORD=$password" ` + --env "DATABASE_NAME=$database" --env "DATABASE_USER=$user" ` + $manager docker stack deploy --compose-file /project/swarm/stack.app.yaml hla-app +Wait-Service 'hla-app_api' '2/2' +Wait-Service 'hla-app_nginx' '1/1' + +Write-Output 'Swarm lab is ready at http://localhost:18080' +Invoke-Checked docker exec $manager docker node ls +Invoke-Checked docker exec $manager docker service ps hla-app_api diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 00000000..d8f971ad --- /dev/null +++ b/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'mtusi-messenger' diff --git a/src/main/java/com/example/mtusi_messenger/MtusiMessengerApplication.java b/src/main/java/com/example/mtusi_messenger/MtusiMessengerApplication.java new file mode 100644 index 00000000..32e25b92 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/MtusiMessengerApplication.java @@ -0,0 +1,18 @@ +package com.example.mtusi_messenger; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.ConfigurableApplicationContext; + +@SpringBootApplication +public class MtusiMessengerApplication { + + public static void main(String[] args) { + ConfigurableApplicationContext context = SpringApplication.run( + MtusiMessengerApplication.class, args); + if (Boolean.parseBoolean(System.getenv("MIGRATE_ONLY"))) { + System.exit(SpringApplication.exit(context)); + } + } + +} diff --git a/src/main/java/com/example/mtusi_messenger/cache/UserCache.java b/src/main/java/com/example/mtusi_messenger/cache/UserCache.java new file mode 100644 index 00000000..04ad8036 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/cache/UserCache.java @@ -0,0 +1,83 @@ +package com.example.mtusi_messenger.cache; + +import com.example.mtusi_messenger.model.User; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.dao.DataAccessException; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.util.Optional; +import java.util.UUID; + +@Component +public class UserCache { + private static final Logger log = LoggerFactory.getLogger(UserCache.class); + private static final String KEY_PREFIX = "messenger:user:"; + + private final StringRedisTemplate redis; + private final ObjectMapper mapper; + private final Duration ttl; + + public UserCache( + StringRedisTemplate redis, + ObjectMapper mapper, + @Value("${messenger.cache.user-ttl-seconds:300}") long ttlSeconds + ) { + if (ttlSeconds <= 0) { + throw new IllegalArgumentException("User cache TTL must be positive"); + } + this.redis = redis; + this.mapper = mapper; + this.ttl = Duration.ofSeconds(ttlSeconds); + } + + public Optional find(UUID userId) { + try { + String value = redis.opsForValue().get(key(userId)); + if (value == null) { + return Optional.empty(); + } + User cachedUser = mapper.readValue(value, User.class); + if (cachedUser == null) { + log.warn("Ignoring null Redis user cache entry for {}", userId); + evict(userId); + return Optional.empty(); + } + return Optional.of(cachedUser); + } catch (DataAccessException e) { + log.warn("Redis user cache is unavailable; using PostgreSQL fallback"); + return Optional.empty(); + } catch (JsonProcessingException e) { + log.warn("Ignoring malformed Redis user cache entry for {}", userId); + evict(userId); + return Optional.empty(); + } + } + + public void put(User user) { + try { + redis.opsForValue().set(key(user.id()), mapper.writeValueAsString(user), ttl); + } catch (DataAccessException e) { + log.warn("Redis user cache is unavailable; user was not cached"); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Cannot serialize user cache entry", e); + } + } + + public void evict(UUID userId) { + try { + redis.delete(key(userId)); + } catch (DataAccessException e) { + log.warn("Redis user cache is unavailable; cache eviction was skipped"); + } + } + + private String key(UUID userId) { + return KEY_PREFIX + userId; + } +} diff --git a/src/main/java/com/example/mtusi_messenger/config/JacksonConfig.java b/src/main/java/com/example/mtusi_messenger/config/JacksonConfig.java new file mode 100644 index 00000000..568a65e2 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/config/JacksonConfig.java @@ -0,0 +1,28 @@ +package com.example.mtusi_messenger.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.boot.jackson.autoconfigure.JsonMapperBuilderCustomizer; + +import tools.jackson.databind.cfg.CoercionAction; +import tools.jackson.databind.cfg.CoercionInputShape; +import tools.jackson.databind.type.LogicalType; + +@Configuration +public class JacksonConfig { + + @Bean + JsonMapperBuilderCustomizer strictStringTypes() { + return builder -> builder.withCoercionConfig( + LogicalType.Textual, + config -> { + config.setCoercion( + CoercionInputShape.Integer, CoercionAction.Fail); + config.setCoercion( + CoercionInputShape.Float, CoercionAction.Fail); + config.setCoercion( + CoercionInputShape.Boolean, CoercionAction.Fail); + } + ); + } +} \ No newline at end of file diff --git a/src/main/java/com/example/mtusi_messenger/config/OpenApiConfig.java b/src/main/java/com/example/mtusi_messenger/config/OpenApiConfig.java new file mode 100644 index 00000000..14cc24f6 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/config/OpenApiConfig.java @@ -0,0 +1,21 @@ +package com.example.mtusi_messenger.config; + + +import io.swagger.v3.oas.annotations.enums.SecuritySchemeType; +import io.swagger.v3.oas.annotations.security.SecurityScheme; +import org.springframework.context.annotation.Configuration; + +@SecurityScheme( + name = "basicAuth", + type = SecuritySchemeType.HTTP, + scheme = "basic" +) +@SecurityScheme( + name = "bearerAuth", + type = SecuritySchemeType.HTTP, + scheme = "bearer", + bearerFormat = "opaque session token" +) +@Configuration +public class OpenApiConfig { +} diff --git a/src/main/java/com/example/mtusi_messenger/config/PasswordConfig.java b/src/main/java/com/example/mtusi_messenger/config/PasswordConfig.java new file mode 100644 index 00000000..98436f17 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/config/PasswordConfig.java @@ -0,0 +1,14 @@ +package com.example.mtusi_messenger.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.crypto.argon2.Argon2PasswordEncoder; +import org.springframework.security.crypto.password.PasswordEncoder; + +@Configuration +public class PasswordConfig { + @Bean + PasswordEncoder passwordEncoder() { + return Argon2PasswordEncoder.defaultsForSpringSecurity_v5_8(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/config/SecurityConfig.java b/src/main/java/com/example/mtusi_messenger/config/SecurityConfig.java new file mode 100644 index 00000000..e38246af --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/config/SecurityConfig.java @@ -0,0 +1,92 @@ +package com.example.mtusi_messenger.config; + +import com.example.mtusi_messenger.security.ApiAuthenticationFilter; +import com.example.mtusi_messenger.security.DatabaseAuthenticationProvider; +import com.example.mtusi_messenger.security.RestAuthenticationEntryPoint; +import com.example.mtusi_messenger.service.AuthenticationService; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpMethod; +import org.springframework.security.authentication.AuthenticationManager; +import org.springframework.security.authentication.ProviderManager; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.web.SecurityFilterChain; +import org.springframework.security.web.authentication.AnonymousAuthenticationFilter; + +@Configuration +@ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.SERVLET) +public class SecurityConfig { + @Bean + AuthenticationManager authenticationManager(DatabaseAuthenticationProvider provider) { + return new ProviderManager(provider); + } + + @Bean + RestAuthenticationEntryPoint basicAuthenticationEntryPoint(ObjectMapper mapper) { + return new RestAuthenticationEntryPoint( + mapper, + "Basic realm=\"messenger\", charset=\"UTF-8\""); + } + + @Bean + RestAuthenticationEntryPoint bearerAuthenticationEntryPoint(ObjectMapper mapper) { + return new RestAuthenticationEntryPoint(mapper, "Bearer realm=\"messenger\""); + } + + @Bean + ApiAuthenticationFilter apiAuthenticationFilter( + AuthenticationManager authenticationManager, + AuthenticationService authenticationService, + @Qualifier("basicAuthenticationEntryPoint") RestAuthenticationEntryPoint basicAuthenticationEntryPoint, + @Qualifier("bearerAuthenticationEntryPoint") RestAuthenticationEntryPoint bearerAuthenticationEntryPoint + ) { + return new ApiAuthenticationFilter( + authenticationManager, + authenticationService, + basicAuthenticationEntryPoint, + bearerAuthenticationEntryPoint); + } + + @Bean + FilterRegistrationBean disableServletFilterRegistration( + ApiAuthenticationFilter filter + ) { + var registration = new FilterRegistrationBean<>(filter); + registration.setEnabled(false); + return registration; + } + + @Bean + SecurityFilterChain securityFilterChain( + HttpSecurity http, + ApiAuthenticationFilter apiAuthenticationFilter, + @Qualifier("bearerAuthenticationEntryPoint") RestAuthenticationEntryPoint bearerAuthenticationEntryPoint + ) throws Exception { + http + .csrf(csrf -> csrf.disable()) + .formLogin(form -> form.disable()) + .httpBasic(basic -> basic.disable()) + .logout(logout -> logout.disable()) + .requestCache(cache -> cache.disable()) + .sessionManagement(session -> session + .sessionCreationPolicy(SessionCreationPolicy.STATELESS)) + .exceptionHandling(errors -> errors + .authenticationEntryPoint(bearerAuthenticationEntryPoint)) + .authorizeHttpRequests(authorize -> authorize + .requestMatchers(HttpMethod.GET, "/health/**").permitAll() + .requestMatchers( + "/swagger-ui.html", + "/swagger-ui/**", + "/v3/api-docs/**" + ).permitAll() + .requestMatchers(HttpMethod.POST, "/api/v1/users").permitAll() + .anyRequest().authenticated()) + .addFilterBefore(apiAuthenticationFilter, AnonymousAuthenticationFilter.class); + return http.build(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/config/WebConfig.java b/src/main/java/com/example/mtusi_messenger/config/WebConfig.java new file mode 100644 index 00000000..ac26bb39 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/config/WebConfig.java @@ -0,0 +1,19 @@ +package com.example.mtusi_messenger.config; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class WebConfig { + @Bean + ObjectMapper objectMapper() { + return JsonMapper.builder().findAndAddModules() + .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .build(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/controller/ChatController.java b/src/main/java/com/example/mtusi_messenger/controller/ChatController.java new file mode 100644 index 00000000..ff02c9bf --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/controller/ChatController.java @@ -0,0 +1,83 @@ +package com.example.mtusi_messenger.controller; + +import com.example.mtusi_messenger.exception.ApiExceptions; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.*; +import com.example.mtusi_messenger.dto.*; +import com.example.mtusi_messenger.model.Chat; +import com.example.mtusi_messenger.service.ChatService; +import com.example.mtusi_messenger.security.SessionPrincipal; +import java.util.List; +import java.util.UUID; + +import static com.example.mtusi_messenger.service.UuidValidator.parse; + +@RestController +@RequestMapping("/api/v1/chats") +@SecurityRequirement(name = "bearerAuth") +@RequiredArgsConstructor +public class ChatController { + + private final ChatService chats; + + @PostMapping + public ResponseEntity create(@AuthenticationPrincipal SessionPrincipal principal, + @Valid @RequestBody CreateChatRequest body + ) { + UUID userId = principal.user().id(); + Chat c = chats.create(body, userId); + return ResponseEntity.status(HttpStatus.CREATED).body(ChatResponse.from(c)); + } + + @GetMapping + public ChatPageResponse list(@AuthenticationPrincipal SessionPrincipal principal, + @RequestParam(required = false) String limit, + @RequestParam(required = false) String cursor + ) { + UUID userId = principal.user().id(); + int parsedLimit = validateLimit(limit); + validateCursorPresence(cursor); + List page = chats.page(userId, parsedLimit, cursor); + boolean hasNext = page.size() > parsedLimit; + if (hasNext) page = page.subList(0, parsedLimit); + return new ChatPageResponse(page.stream().map(ChatResponse::from).toList(), page.isEmpty() ? null : + chats.nextCursor(userId, page.get(page.size() - 1), hasNext)); + } + + @GetMapping("/{chatId}") + public ChatResponse get(@AuthenticationPrincipal SessionPrincipal principal, + @PathVariable String chatId + ) { + UUID userId = principal.user().id(); + return ChatResponse.from(chats.getAccessible(parse(chatId), userId)); + } + + private static int validateLimit(String limit) { + if (limit == null) { + return 50; + } + if (!limit.matches("[0-9]+")) { + throw ApiExceptions.invalid("Invalid limit"); + } + try { + int value = Integer.parseInt(limit); + + if (value < 1 || value > 100) { + throw ApiExceptions.invalid("Invalid limit"); + } + + return value; + } catch (NumberFormatException ex) { + throw ApiExceptions.invalid("Invalid limit"); + } + } + + private static void validateCursorPresence(String c) { + if (c != null && c.isBlank()) throw ApiExceptions.invalid("Invalid cursor"); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/controller/HealthController.java b/src/main/java/com/example/mtusi_messenger/controller/HealthController.java new file mode 100644 index 00000000..2b9abc35 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/controller/HealthController.java @@ -0,0 +1,58 @@ +package com.example.mtusi_messenger.controller; + +import org.springframework.http.ResponseEntity; +import org.springframework.dao.DataAccessException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.RedisCallback; +import org.springframework.boot.availability.ApplicationAvailability; +import org.springframework.boot.availability.ReadinessState; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; +import com.example.mtusi_messenger.dto.HealthResponse; +import com.example.mtusi_messenger.exception.ApiExceptions; + +@RestController +public class HealthController { + private final JdbcTemplate jdbc; + private final StringRedisTemplate redis; + private final ApplicationAvailability availability; + + public HealthController( + JdbcTemplate jdbc, + StringRedisTemplate redis, + ApplicationAvailability availability + ) { + this.jdbc = jdbc; + this.redis = redis; + this.availability = availability; + } + + @GetMapping("/health/live") + public ResponseEntity live() { + return ResponseEntity.ok(new HealthResponse("ok")); + } + + @GetMapping("/health/ready") + public ResponseEntity ready() { + if (availability.getReadinessState() != ReadinessState.ACCEPTING_TRAFFIC) { + throw ApiExceptions.unavailable("Application is not accepting traffic"); + } + try { + // Warm the lazy Lettuce connection on every replica. Redis remains + // an optional cache: failure is ignored and business reads fall + // back to PostgreSQL. + redis.execute((RedisCallback) connection -> connection.ping()); + } catch (DataAccessException ignored) { + // Optional cache unavailable; PostgreSQL remains authoritative. + } + try { + // A schema-level probe prevents a replica from becoming ready + // before the separately executed migration job has completed. + jdbc.queryForObject("SELECT count(*) FROM message_idempotency", Long.class); + return ResponseEntity.ok(new HealthResponse("ok")); + } catch (DataAccessException e) { + throw ApiExceptions.unavailable("Database is unavailable"); + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/controller/MessageController.java b/src/main/java/com/example/mtusi_messenger/controller/MessageController.java new file mode 100644 index 00000000..e26c4cb2 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/controller/MessageController.java @@ -0,0 +1,79 @@ +package com.example.mtusi_messenger.controller; + +import com.example.mtusi_messenger.exception.ApiExceptions; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.*; +import com.example.mtusi_messenger.dto.*; +import com.example.mtusi_messenger.model.Message; +import com.example.mtusi_messenger.service.MessageService; +import com.example.mtusi_messenger.security.SessionPrincipal; +import java.util.List; +import java.util.UUID; + +import static com.example.mtusi_messenger.service.UuidValidator.parse; + +@RequiredArgsConstructor +@RestController +@RequestMapping("/api/v1/chats/{chatId}/messages") +@SecurityRequirement(name = "bearerAuth") +public class MessageController { + private static final String IDEMPOTENCY_KEY_PATTERN = "[A-Za-z0-9._:-]{1,128}"; + private final MessageService messages; + + @PostMapping + public ResponseEntity create(@AuthenticationPrincipal SessionPrincipal principal, + @PathVariable String chatId, + @RequestHeader(name = "Idempotency-Key", required = false) String idempotencyKey, + @Valid @RequestBody CreateMessageRequest body + ) { + if (idempotencyKey == null || !idempotencyKey.matches(IDEMPOTENCY_KEY_PATTERN)) { + throw ApiExceptions.invalid("Invalid or missing Idempotency-Key"); + } + UUID userId = principal.user().id(); + UUID parsedChatId = parse(chatId); + Message m = messages.create(parsedChatId, userId, body, idempotencyKey); + return ResponseEntity.status(HttpStatus.CREATED).body(MessageResponse.from(m)); + } + + @GetMapping + public MessagePageResponse list(@AuthenticationPrincipal SessionPrincipal principal, + @PathVariable String chatId, + @RequestParam(required = false) String limit, + @RequestParam(required=false) String cursor + ) { + UUID userId = principal.user().id(); + UUID parsedChatId = parse(chatId); + int parsedLimit = validateLimit(limit); + if (cursor != null && cursor.isBlank()) throw ApiExceptions.invalid("Invalid cursor"); + List page = messages.page(parsedChatId, userId, parsedLimit, cursor); + boolean hasNext = page.size() > parsedLimit; if (hasNext) page = page.subList(0, parsedLimit); + return new MessagePageResponse(page.stream().map(MessageResponse::from).toList(), + page.isEmpty() ? null : messages.nextCursor(parsedChatId, page.get(page.size()-1), hasNext)); + } + + private static int validateLimit(String limit) { + if (limit == null) { + return 50; + } + if (!limit.matches("[0-9]+")) { + throw ApiExceptions.invalid("Invalid limit"); + } + try { + int value = Integer.parseInt(limit); + + if (value < 1 || value > 100) { + throw ApiExceptions.invalid("Invalid limit"); + } + + return value; + } catch (NumberFormatException ex) { + throw ApiExceptions.invalid("Invalid limit"); + } + } + +} diff --git a/src/main/java/com/example/mtusi_messenger/controller/SessionController.java b/src/main/java/com/example/mtusi_messenger/controller/SessionController.java new file mode 100644 index 00000000..f79528f3 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/controller/SessionController.java @@ -0,0 +1,41 @@ +package com.example.mtusi_messenger.controller; + +import com.example.mtusi_messenger.dto.SessionResponse; +import com.example.mtusi_messenger.dto.UserResponse; +import com.example.mtusi_messenger.service.AuthenticationService; +import com.example.mtusi_messenger.model.User; +import com.example.mtusi_messenger.security.SessionPrincipal; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/v1/auth/sessions") +@RequiredArgsConstructor +public class SessionController { + private final AuthenticationService authentication; + + @PostMapping + @SecurityRequirement(name = "basicAuth") + public ResponseEntity create(@AuthenticationPrincipal User user) { + var issued = authentication.issueSession(user); + return ResponseEntity.status(HttpStatus.CREATED).body(new SessionResponse( + issued.token(), + "Bearer", + issued.expiresAt(), + UserResponse.from(issued.user()))); + } + + @DeleteMapping("/current") + @SecurityRequirement(name = "bearerAuth") + public ResponseEntity revoke(@AuthenticationPrincipal SessionPrincipal principal) { + authentication.revoke(principal); + return ResponseEntity.noContent().build(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/controller/UserController.java b/src/main/java/com/example/mtusi_messenger/controller/UserController.java new file mode 100644 index 00000000..0d2d9cf6 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/controller/UserController.java @@ -0,0 +1,32 @@ +package com.example.mtusi_messenger.controller; + +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.*; +import com.example.mtusi_messenger.dto.*; +import com.example.mtusi_messenger.service.UserService; +import com.example.mtusi_messenger.security.SessionPrincipal; + +@RestController +@RequestMapping("/api/v1/users") +@RequiredArgsConstructor +public class UserController { + + private final UserService users; + + @PostMapping + public ResponseEntity create(@Valid @RequestBody CreateUserRequest request) { + var user = users.create(request); + return ResponseEntity.status(HttpStatus.CREATED).body(UserResponse.from(user)); + } + + @GetMapping("/me") + @SecurityRequirement(name = "bearerAuth") + public UserResponse me(@AuthenticationPrincipal SessionPrincipal principal) { + return UserResponse.from(principal.user()); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/dto/ChatPageResponse.java b/src/main/java/com/example/mtusi_messenger/dto/ChatPageResponse.java new file mode 100644 index 00000000..e1d54bf2 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/ChatPageResponse.java @@ -0,0 +1,8 @@ +package com.example.mtusi_messenger.dto; + +import java.util.List; + +public record ChatPageResponse( + List items, + String next_cursor +) {} diff --git a/src/main/java/com/example/mtusi_messenger/dto/ChatResponse.java b/src/main/java/com/example/mtusi_messenger/dto/ChatResponse.java new file mode 100644 index 00000000..96b2abb4 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/ChatResponse.java @@ -0,0 +1,18 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.model.Chat; +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record ChatResponse(UUID id, + String title, + List member_ids, + Instant created_at) { + public static ChatResponse from(Chat c) { + return new ChatResponse( + c.id(), + c.title(), + c.memberIds(), + c.createdAt()); } +} diff --git a/src/main/java/com/example/mtusi_messenger/dto/CreateChatRequest.java b/src/main/java/com/example/mtusi_messenger/dto/CreateChatRequest.java new file mode 100644 index 00000000..3f792346 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/CreateChatRequest.java @@ -0,0 +1,19 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.web.NotUnicodeBlank; +import com.example.mtusi_messenger.web.UnicodeSize; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.Size; +import java.util.List; +import java.util.UUID; + +public record CreateChatRequest( + @NotUnicodeBlank + @UnicodeSize(min = 1, max = 100) + String title, + + @NotEmpty + @Size(min = 1, max = 100) + List member_ids +) {} diff --git a/src/main/java/com/example/mtusi_messenger/dto/CreateMessageRequest.java b/src/main/java/com/example/mtusi_messenger/dto/CreateMessageRequest.java new file mode 100644 index 00000000..6900ecab --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/CreateMessageRequest.java @@ -0,0 +1,8 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.web.NotUnicodeBlank; +import com.example.mtusi_messenger.web.UnicodeSize; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; + +public record CreateMessageRequest( @NotUnicodeBlank @UnicodeSize(min = 1, max = 4000) String text) {} diff --git a/src/main/java/com/example/mtusi_messenger/dto/CreateUserRequest.java b/src/main/java/com/example/mtusi_messenger/dto/CreateUserRequest.java new file mode 100644 index 00000000..6d6e5a91 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/CreateUserRequest.java @@ -0,0 +1,20 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.web.NotUnicodeBlank; +import com.example.mtusi_messenger.web.UnicodeSize; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Pattern; + +public record CreateUserRequest( + @NotBlank + @Pattern(regexp = "[a-z0-9_]{3,32}") + String username, + + @NotUnicodeBlank + @UnicodeSize(min = 1, max = 100) + String display_name, + + @NotUnicodeBlank + @UnicodeSize(min = 12, max = 128) + String password +) {} diff --git a/src/main/java/com/example/mtusi_messenger/dto/ErrorResponse.java b/src/main/java/com/example/mtusi_messenger/dto/ErrorResponse.java new file mode 100644 index 00000000..4b1a2a92 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/ErrorResponse.java @@ -0,0 +1,8 @@ +package com.example.mtusi_messenger.dto; + +public record ErrorResponse(ErrorBody error) { + public record ErrorBody(String code, + String message, + String request_id) + {} +} diff --git a/src/main/java/com/example/mtusi_messenger/dto/HealthResponse.java b/src/main/java/com/example/mtusi_messenger/dto/HealthResponse.java new file mode 100644 index 00000000..9630b107 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/HealthResponse.java @@ -0,0 +1,3 @@ +package com.example.mtusi_messenger.dto; + +public record HealthResponse(String status) {} diff --git a/src/main/java/com/example/mtusi_messenger/dto/MessagePageResponse.java b/src/main/java/com/example/mtusi_messenger/dto/MessagePageResponse.java new file mode 100644 index 00000000..db72eee8 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/MessagePageResponse.java @@ -0,0 +1,8 @@ +package com.example.mtusi_messenger.dto; + +import java.util.List; + +public record MessagePageResponse( + List items, + String next_cursor +) {} diff --git a/src/main/java/com/example/mtusi_messenger/dto/MessageResponse.java b/src/main/java/com/example/mtusi_messenger/dto/MessageResponse.java new file mode 100644 index 00000000..d780c354 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/MessageResponse.java @@ -0,0 +1,19 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.model.Message; +import java.time.Instant; +import java.util.UUID; + +public record MessageResponse(UUID id, + UUID chat_id, + UUID sender_id, + String text, + Instant created_at) { + public static MessageResponse from(Message m) { + return new MessageResponse(m.id(), + m.chatId(), + m.senderId(), + m.text(), + m.createdAt()); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/dto/SessionResponse.java b/src/main/java/com/example/mtusi_messenger/dto/SessionResponse.java new file mode 100644 index 00000000..64b54f99 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/SessionResponse.java @@ -0,0 +1,10 @@ +package com.example.mtusi_messenger.dto; + +import java.time.Instant; + +public record SessionResponse( + String token, + String token_type, + Instant expires_at, + UserResponse user +) {} diff --git a/src/main/java/com/example/mtusi_messenger/dto/UserResponse.java b/src/main/java/com/example/mtusi_messenger/dto/UserResponse.java new file mode 100644 index 00000000..e84bcb71 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/dto/UserResponse.java @@ -0,0 +1,18 @@ +package com.example.mtusi_messenger.dto; + +import com.example.mtusi_messenger.model.User; +import java.time.Instant; +import java.util.UUID; + +public record UserResponse(UUID id, + String username, + String display_name, + Instant created_at) { + public static UserResponse from(User u) { + return new UserResponse( + u.id(), + u.username(), + u.displayName(), + u.createdAt()); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/exception/ApiException.java b/src/main/java/com/example/mtusi_messenger/exception/ApiException.java new file mode 100644 index 00000000..2c07f225 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/exception/ApiException.java @@ -0,0 +1,14 @@ +package com.example.mtusi_messenger.exception; + +import org.springframework.http.HttpStatus; + +public class ApiException extends RuntimeException { + private final String code; private final HttpStatus status; private final String authenticate; + public ApiException(String code, HttpStatus status, String message) { this(code, status, message, null); } + public ApiException(String code, HttpStatus status, String message, String authenticate) { + super(message); this.code = code; this.status = status; this.authenticate = authenticate; + } + public String code() { return code; } + public HttpStatus status() { return status; } + public String authenticate() { return authenticate; } +} diff --git a/src/main/java/com/example/mtusi_messenger/exception/ApiExceptions.java b/src/main/java/com/example/mtusi_messenger/exception/ApiExceptions.java new file mode 100644 index 00000000..cf671bf9 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/exception/ApiExceptions.java @@ -0,0 +1,14 @@ +package com.example.mtusi_messenger.exception; + +import org.springframework.http.HttpStatus; + +public final class ApiExceptions { + private ApiExceptions() {} + public static ApiException invalid(String m) { return new ApiException("invalid_request", HttpStatus.BAD_REQUEST, m); } + public static ApiException unauthorized(String m) { return unauthorizedBearer(m); } + public static ApiException unauthorizedBasic(String m) { return new ApiException("unauthorized", HttpStatus.UNAUTHORIZED, m, "Basic realm=\"messenger\", charset=\"UTF-8\""); } + public static ApiException unauthorizedBearer(String m) { return new ApiException("unauthorized", HttpStatus.UNAUTHORIZED, m, "Bearer realm=\"messenger\""); } + public static ApiException notFound(String m) { return new ApiException("not_found", HttpStatus.NOT_FOUND, m); } + public static ApiException conflict(String m) { return new ApiException("conflict", HttpStatus.CONFLICT, m); } + public static ApiException unavailable(String m) { return new ApiException("unavailable", HttpStatus.SERVICE_UNAVAILABLE, m); } +} diff --git a/src/main/java/com/example/mtusi_messenger/model/Chat.java b/src/main/java/com/example/mtusi_messenger/model/Chat.java new file mode 100644 index 00000000..829d99e5 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/Chat.java @@ -0,0 +1,11 @@ +package com.example.mtusi_messenger.model; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record Chat(UUID id, String title, List memberIds, Instant createdAt) { + public Chat { + memberIds = List.copyOf(memberIds); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/model/IdempotentMessage.java b/src/main/java/com/example/mtusi_messenger/model/IdempotentMessage.java new file mode 100644 index 00000000..424d7a33 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/IdempotentMessage.java @@ -0,0 +1,3 @@ +package com.example.mtusi_messenger.model; + +public record IdempotentMessage(byte[] requestHash, Message message) {} diff --git a/src/main/java/com/example/mtusi_messenger/model/Message.java b/src/main/java/com/example/mtusi_messenger/model/Message.java new file mode 100644 index 00000000..a52586eb --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/Message.java @@ -0,0 +1,6 @@ +package com.example.mtusi_messenger.model; + +import java.time.Instant; +import java.util.UUID; + +public record Message(UUID id, UUID chatId, UUID senderId, String text, Instant createdAt) {} diff --git a/src/main/java/com/example/mtusi_messenger/model/MessageKey.java b/src/main/java/com/example/mtusi_messenger/model/MessageKey.java new file mode 100644 index 00000000..545fd607 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/MessageKey.java @@ -0,0 +1,35 @@ +package com.example.mtusi_messenger.model; + +import java.time.Instant; +import java.util.UUID; + +public record MessageKey(Instant createdAt, UUID id) + implements Comparable { + + private static int compareUuidV7(UUID a, UUID b) { + int result = Long.compareUnsigned( + a.getMostSignificantBits(), + b.getMostSignificantBits() + ); + + if (result != 0) { + return result; + } + + return Long.compareUnsigned( + a.getLeastSignificantBits(), + b.getLeastSignificantBits() + ); + } + + @Override + public int compareTo(MessageKey other) { + int result = createdAt.compareTo(other.createdAt); + + if (result != 0) { + return result; + } + + return compareUuidV7(id, other.id); + } +} \ No newline at end of file diff --git a/src/main/java/com/example/mtusi_messenger/model/Session.java b/src/main/java/com/example/mtusi_messenger/model/Session.java new file mode 100644 index 00000000..27aa2fbe --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/Session.java @@ -0,0 +1,15 @@ +package com.example.mtusi_messenger.model; + +import java.time.Instant; +import java.util.UUID; + +public record Session(UUID id, UUID userId, byte[] tokenHash, Instant createdAt, Instant expiresAt) { + public Session { + tokenHash = tokenHash.clone(); + } + + @Override + public byte[] tokenHash() { + return tokenHash.clone(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/model/User.java b/src/main/java/com/example/mtusi_messenger/model/User.java new file mode 100644 index 00000000..1fefb17c --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/model/User.java @@ -0,0 +1,6 @@ +package com.example.mtusi_messenger.model; + +import java.time.Instant; +import java.util.UUID; + +public record User(UUID id, String username, String displayName, Instant createdAt) {} diff --git a/src/main/java/com/example/mtusi_messenger/repository/ChatRepository.java b/src/main/java/com/example/mtusi_messenger/repository/ChatRepository.java new file mode 100644 index 00000000..66085e44 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/ChatRepository.java @@ -0,0 +1,14 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Chat; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +public interface ChatRepository { + Chat save(Chat chat); + Optional findById(UUID id); + boolean isMember(UUID chatId, UUID userId); + List findAllForUser(UUID userId); + List findPageForUser(UUID userId, UUID afterId, java.time.Instant afterCreatedAt, int limitPlusOne); +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/MessageRepository.java b/src/main/java/com/example/mtusi_messenger/repository/MessageRepository.java new file mode 100644 index 00000000..23ea4814 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/MessageRepository.java @@ -0,0 +1,18 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Message; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +public interface MessageRepository { + Message save(Message message); + void lockIdempotencyScope(UUID userId, String method, String path, String key); + void deleteExpiredIdempotency(UUID userId, String method, String path, String key, Instant expiresBefore); + Optional findIdempotent( + UUID userId, String method, String path, String key); + void saveIdempotency(UUID userId, String method, String path, String key, + byte[] requestHash, UUID messageId, Instant createdAt); + List findPage(UUID chatId, Instant afterCreatedAt, UUID afterId, int limitPlusOne); +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/PostgresChatRepository.java b/src/main/java/com/example/mtusi_messenger/repository/PostgresChatRepository.java new file mode 100644 index 00000000..17d825d7 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/PostgresChatRepository.java @@ -0,0 +1,106 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Chat; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class PostgresChatRepository implements ChatRepository { + private final JdbcTemplate jdbc; + + public PostgresChatRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + @Transactional(timeout = 5) + public Chat save(Chat chat) { + jdbc.update(""" + INSERT INTO chats (id, title, created_at) + VALUES (?, ?, ?) + """, chat.id(), chat.title(), Timestamp.from(chat.createdAt())); + + for (UUID memberId : chat.memberIds()) { + jdbc.update(""" + INSERT INTO chat_members (chat_id, user_id) + VALUES (?, ?) + """, chat.id(), memberId); + } + return chat; + } + + @Override + public Optional findById(UUID id) { + return jdbc.query(""" + SELECT id, title, created_at + FROM chats + WHERE id = ? + """, (rs, rowNum) -> new ChatRow( + rs.getObject("id", UUID.class), + rs.getString("title"), + rs.getTimestamp("created_at").toInstant()), id) + .stream().findFirst() + .map(this::toChat); + } + + @Override + public boolean isMember(UUID chatId, UUID userId) { + return Boolean.TRUE.equals(jdbc.queryForObject(""" + SELECT EXISTS ( + SELECT 1 FROM chat_members + WHERE chat_id = ? AND user_id = ? + ) + """, Boolean.class, chatId, userId)); + } + + @Override + public List findAllForUser(UUID userId) { + return findPageForUser(userId, null, null, Integer.MAX_VALUE); + } + + @Override + public List findPageForUser(UUID userId, UUID afterId, Instant afterCreatedAt, int limitPlusOne) { + StringBuilder sql = new StringBuilder(""" + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE cm.user_id = ? + """); + List parameters = new ArrayList<>(); + parameters.add(userId); + + if (afterCreatedAt != null) { + sql.append(" AND (c.created_at, c.id) > (?, ?) "); + parameters.add(Timestamp.from(afterCreatedAt)); + parameters.add(afterId); + } + sql.append(" ORDER BY c.created_at, c.id LIMIT ? "); + parameters.add(limitPlusOne); + + return jdbc.query(sql.toString(), (rs, rowNum) -> new ChatRow( + rs.getObject("id", UUID.class), + rs.getString("title"), + rs.getTimestamp("created_at").toInstant()), parameters.toArray()) + .stream().map(this::toChat).toList(); + } + + private Chat toChat(ChatRow row) { + List members = jdbc.queryForList(""" + SELECT user_id + FROM chat_members + WHERE chat_id = ? + ORDER BY joined_at, user_id + """, UUID.class, row.id()); + return new Chat(row.id(), row.title(), members, row.createdAt()); + } + + private record ChatRow(UUID id, String title, Instant createdAt) {} +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/PostgresMessageRepository.java b/src/main/java/com/example/mtusi_messenger/repository/PostgresMessageRepository.java new file mode 100644 index 00000000..0008a7a3 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/PostgresMessageRepository.java @@ -0,0 +1,134 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Message; +import com.example.mtusi_messenger.model.IdempotentMessage; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class PostgresMessageRepository implements MessageRepository { + private final JdbcTemplate jdbc; + + public PostgresMessageRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + public Message save(Message message) { + jdbc.update(""" + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES (?, ?, ?, ?, ?) + """, message.id(), message.chatId(), message.senderId(), message.text(), + Timestamp.from(message.createdAt())); + return message; + } + + @Override + public void lockIdempotencyScope(UUID userId, String method, String path, String key) { + String scope = userId + "\n" + method + "\n" + path + "\n" + key; + jdbc.query( + "SELECT pg_advisory_xact_lock(hashtextextended(?, 0))", + rs -> { + rs.next(); + return null; + }, + scope); + } + + @Override + public void deleteExpiredIdempotency( + UUID userId, + String method, + String path, + String key, + Instant expiresBefore + ) { + jdbc.update(""" + DELETE FROM message_idempotency + WHERE user_id = ? + AND method = ? + AND request_path = ? + AND idempotency_key = ? + AND created_at < ? + """, userId, method, path, key, Timestamp.from(expiresBefore)); + } + + @Override + public Optional findIdempotent( + UUID userId, + String method, + String path, + String key + ) { + return jdbc.query(""" + SELECT i.request_hash, + m.id, m.chat_id, m.sender_id, m.text, m.created_at + FROM message_idempotency i + JOIN messages m ON m.id = i.message_id + WHERE i.user_id = ? + AND i.method = ? + AND i.request_path = ? + AND i.idempotency_key = ? + """, (rs, rowNum) -> new IdempotentMessage( + rs.getBytes("request_hash"), + new Message( + rs.getObject("id", UUID.class), + rs.getObject("chat_id", UUID.class), + rs.getObject("sender_id", UUID.class), + rs.getString("text"), + rs.getTimestamp("created_at").toInstant())), + userId, method, path, key).stream().findFirst(); + } + + @Override + public void saveIdempotency( + UUID userId, + String method, + String path, + String key, + byte[] requestHash, + UUID messageId, + Instant createdAt + ) { + jdbc.update(""" + INSERT INTO message_idempotency + (user_id, method, request_path, idempotency_key, + request_hash, message_id, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, userId, method, path, key, requestHash, messageId, + Timestamp.from(createdAt)); + } + + @Override + public List findPage(UUID chatId, Instant afterCreatedAt, UUID afterId, int limitPlusOne) { + StringBuilder sql = new StringBuilder(""" + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = ? + """); + List parameters = new ArrayList<>(); + parameters.add(chatId); + + if (afterCreatedAt != null) { + sql.append(" AND (created_at, id) > (?, ?) "); + parameters.add(Timestamp.from(afterCreatedAt)); + parameters.add(afterId); + } + sql.append(" ORDER BY created_at, id LIMIT ? "); + parameters.add(limitPlusOne); + + return jdbc.query(sql.toString(), (rs, rowNum) -> new Message( + rs.getObject("id", UUID.class), + rs.getObject("chat_id", UUID.class), + rs.getObject("sender_id", UUID.class), + rs.getString("text"), + rs.getTimestamp("created_at").toInstant()), parameters.toArray()); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/PostgresSessionRepository.java b/src/main/java/com/example/mtusi_messenger/repository/PostgresSessionRepository.java new file mode 100644 index 00000000..d0b210b0 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/PostgresSessionRepository.java @@ -0,0 +1,51 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Session; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.Timestamp; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class PostgresSessionRepository implements SessionRepository { + private final JdbcTemplate jdbc; + + public PostgresSessionRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + public Session save(Session session) { + jdbc.update(""" + INSERT INTO sessions (id, user_id, token_hash, created_at, expires_at) + VALUES (?, ?, ?, ?, ?) + """, session.id(), session.userId(), session.tokenHash(), + Timestamp.from(session.createdAt()), Timestamp.from(session.expiresAt())); + return session; + } + + @Override + public Optional findActiveUserId(byte[] tokenHash) { + return jdbc.query(""" + SELECT user_id + FROM sessions + WHERE token_hash = ? + AND revoked_at IS NULL + AND expires_at > CURRENT_TIMESTAMP + """, (rs, rowNum) -> rs.getObject("user_id", UUID.class), tokenHash) + .stream().findFirst(); + } + + @Override + public boolean revoke(byte[] tokenHash) { + return jdbc.update(""" + UPDATE sessions + SET revoked_at = CURRENT_TIMESTAMP + WHERE token_hash = ? + AND revoked_at IS NULL + AND expires_at > CURRENT_TIMESTAMP + """, tokenHash) == 1; + } +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/PostgresUserRepository.java b/src/main/java/com/example/mtusi_messenger/repository/PostgresUserRepository.java new file mode 100644 index 00000000..e096b480 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/PostgresUserRepository.java @@ -0,0 +1,72 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.User; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.Timestamp; +import java.util.Optional; +import java.util.UUID; + +@Repository +public class PostgresUserRepository implements UserRepository { + private final JdbcTemplate jdbc; + + public PostgresUserRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + public User save(User user, String passwordHash) { + jdbc.update(""" + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (?, ?, ?, ?, ?) + """, user.id(), user.username(), user.displayName(), passwordHash, + Timestamp.from(user.createdAt())); + return user; + } + + @Override + public Optional findById(UUID id) { + return jdbc.query(""" + SELECT id, username, display_name, created_at + FROM users + WHERE id = ? + """, (rs, rowNum) -> new User( + rs.getObject("id", UUID.class), + rs.getString("username"), + rs.getString("display_name"), + rs.getTimestamp("created_at").toInstant()), id) + .stream().findFirst(); + } + + @Override + public Optional findByUsername(String username) { + return jdbc.query(""" + SELECT id, username, display_name, created_at + FROM users + WHERE username = ? + """, (rs, rowNum) -> new User( + rs.getObject("id", UUID.class), + rs.getString("username"), + rs.getString("display_name"), + rs.getTimestamp("created_at").toInstant()), username) + .stream().findFirst(); + } + + @Override + public Optional findCredentialsByUsername(String username) { + return jdbc.query(""" + SELECT id, username, display_name, password_hash, created_at + FROM users + WHERE username = ? + """, (rs, rowNum) -> new UserCredentials( + new User( + rs.getObject("id", UUID.class), + rs.getString("username"), + rs.getString("display_name"), + rs.getTimestamp("created_at").toInstant()), + rs.getString("password_hash")), username) + .stream().findFirst(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/SessionRepository.java b/src/main/java/com/example/mtusi_messenger/repository/SessionRepository.java new file mode 100644 index 00000000..801c9cf4 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/SessionRepository.java @@ -0,0 +1,12 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.Session; + +import java.util.Optional; +import java.util.UUID; + +public interface SessionRepository { + Session save(Session session); + Optional findActiveUserId(byte[] tokenHash); + boolean revoke(byte[] tokenHash); +} diff --git a/src/main/java/com/example/mtusi_messenger/repository/UserRepository.java b/src/main/java/com/example/mtusi_messenger/repository/UserRepository.java new file mode 100644 index 00000000..e4861960 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/repository/UserRepository.java @@ -0,0 +1,14 @@ +package com.example.mtusi_messenger.repository; + +import com.example.mtusi_messenger.model.User; +import java.util.Optional; +import java.util.UUID; + +public interface UserRepository { + User save(User user, String passwordHash); + Optional findById(UUID id); + Optional findByUsername(String username); + Optional findCredentialsByUsername(String username); + + record UserCredentials(User user, String passwordHash) {} +} diff --git a/src/main/java/com/example/mtusi_messenger/security/ApiAuthenticationFilter.java b/src/main/java/com/example/mtusi_messenger/security/ApiAuthenticationFilter.java new file mode 100644 index 00000000..213d8a35 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/security/ApiAuthenticationFilter.java @@ -0,0 +1,130 @@ +package com.example.mtusi_messenger.security; + +import com.example.mtusi_messenger.service.AuthenticationService; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.http.HttpMethod; +import org.springframework.security.authentication.AuthenticationManager; +import org.springframework.security.authentication.BadCredentialsException; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.AuthenticationException; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Enumeration; +import java.util.List; + +public class ApiAuthenticationFilter extends OncePerRequestFilter { + private static final String AUTHORIZATION = "Authorization"; + private static final String SESSION_PATH = "/api/v1/auth/sessions"; + + private final AuthenticationManager authenticationManager; + private final AuthenticationService sessions; + private final RestAuthenticationEntryPoint basicEntryPoint; + private final RestAuthenticationEntryPoint bearerEntryPoint; + + public ApiAuthenticationFilter( + AuthenticationManager authenticationManager, + AuthenticationService sessions, + RestAuthenticationEntryPoint basicEntryPoint, + RestAuthenticationEntryPoint bearerEntryPoint + ) { + this.authenticationManager = authenticationManager; + this.sessions = sessions; + this.basicEntryPoint = basicEntryPoint; + this.bearerEntryPoint = bearerEntryPoint; + } + + @Override + protected void doFilterInternal( + HttpServletRequest request, + HttpServletResponse response, + FilterChain filterChain + ) throws ServletException, IOException { + boolean basicLogin = HttpMethod.POST.matches(request.getMethod()) + && SESSION_PATH.equals(request.getRequestURI()); + if (isPublicRequest(request) && !basicLogin) { + filterChain.doFilter(request, response); + return; + } + + try { + Authentication authentication = basicLogin + ? authenticateBasic(request) + : authenticateBearer(request); + var context = SecurityContextHolder.createEmptyContext(); + context.setAuthentication(authentication); + SecurityContextHolder.setContext(context); + filterChain.doFilter(request, response); + } catch (AuthenticationException e) { + SecurityContextHolder.clearContext(); + (basicLogin ? basicEntryPoint : bearerEntryPoint) + .commence(request, response, e); + } finally { + SecurityContextHolder.clearContext(); + } + } + + private Authentication authenticateBasic(HttpServletRequest request) { + String header = singleAuthorizationHeader(request); + if (!header.startsWith("Basic ") || header.length() == 6) { + throw new BadCredentialsException("Invalid Basic credentials"); + } + try { + byte[] decoded = Base64.getDecoder().decode(header.substring(6)); + String credentials = new String(decoded, StandardCharsets.UTF_8); + int separator = credentials.indexOf(':'); + if (separator <= 0) { + throw new BadCredentialsException("Invalid Basic credentials"); + } + return authenticationManager.authenticate( + UsernamePasswordAuthenticationToken.unauthenticated( + credentials.substring(0, separator), + credentials.substring(separator + 1))); + } catch (IllegalArgumentException e) { + throw new BadCredentialsException("Invalid Basic credentials", e); + } + } + + private Authentication authenticateBearer(HttpServletRequest request) { + String header = singleAuthorizationHeader(request); + if (!header.startsWith("Bearer ") || header.length() == 7) { + throw new BadCredentialsException("Invalid Bearer token"); + } + String token = header.substring(7); + if (!token.matches("[A-Za-z0-9_-]{43}")) { + throw new BadCredentialsException("Invalid Bearer token"); + } + SessionPrincipal principal = sessions.authenticateBearer(token); + return UsernamePasswordAuthenticationToken.authenticated(principal, null, List.of()); + } + + private String singleAuthorizationHeader(HttpServletRequest request) { + Enumeration values = request.getHeaders(AUTHORIZATION); + if (values == null || !values.hasMoreElements()) { + throw new BadCredentialsException("Authentication required"); + } + String value = values.nextElement(); + if (values.hasMoreElements()) { + throw new BadCredentialsException("Multiple Authorization headers"); + } + return value; + } + + private boolean isPublicRequest(HttpServletRequest request) { + String path = request.getRequestURI(); + return path.startsWith("/health/") + || path.equals("/swagger-ui.html") + || path.startsWith("/swagger-ui/") + || path.equals("/v3/api-docs") + || path.startsWith("/v3/api-docs/") + || (HttpMethod.POST.matches(request.getMethod()) + && "/api/v1/users".equals(path)); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/security/DatabaseAuthenticationProvider.java b/src/main/java/com/example/mtusi_messenger/security/DatabaseAuthenticationProvider.java new file mode 100644 index 00000000..0a0c04da --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/security/DatabaseAuthenticationProvider.java @@ -0,0 +1,63 @@ +package com.example.mtusi_messenger.security; + +import com.example.mtusi_messenger.repository.UserRepository; +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.authentication.AuthenticationServiceException; +import org.springframework.security.authentication.BadCredentialsException; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.AuthenticationException; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.stereotype.Component; +import org.springframework.dao.DataAccessException; + +import java.util.List; + +@Component +public class DatabaseAuthenticationProvider implements AuthenticationProvider { + private final UserRepository users; + private final PasswordEncoder passwordEncoder; + private final String dummyPasswordHash; + + public DatabaseAuthenticationProvider(UserRepository users, PasswordEncoder passwordEncoder) { + this.users = users; + this.passwordEncoder = passwordEncoder; + this.dummyPasswordHash = passwordEncoder.encode("dummy-password-used-only-for-timing"); + } + + @Override + public Authentication authenticate(Authentication authentication) throws AuthenticationException { + String username = authentication.getName(); + String password = String.valueOf(authentication.getCredentials()); + final java.util.Optional stored; + try { + stored = users.findCredentialsByUsername(username); + } catch (DataAccessException e) { + throw new AuthenticationServiceException("Authentication database is unavailable", e); + } + String passwordHash = stored.map(UserRepository.UserCredentials::passwordHash) + .filter(hash -> hash.startsWith("$argon2")) + .orElse(dummyPasswordHash); + + boolean matches; + try { + matches = passwordEncoder.matches(password, passwordHash); + } catch (IllegalArgumentException e) { + matches = false; + } + + if (stored.isEmpty() || !matches) { + throw new BadCredentialsException("Invalid credentials"); + } + + return UsernamePasswordAuthenticationToken.authenticated( + stored.get().user(), + null, + List.of()); + } + + @Override + public boolean supports(Class authentication) { + return UsernamePasswordAuthenticationToken.class.isAssignableFrom(authentication); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/security/RestAuthenticationEntryPoint.java b/src/main/java/com/example/mtusi_messenger/security/RestAuthenticationEntryPoint.java new file mode 100644 index 00000000..4442f35c --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/security/RestAuthenticationEntryPoint.java @@ -0,0 +1,46 @@ +package com.example.mtusi_messenger.security; + +import com.example.mtusi_messenger.dto.ErrorResponse; +import com.example.mtusi_messenger.web.RequestIdFilter; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.http.MediaType; +import org.springframework.security.core.AuthenticationException; +import org.springframework.security.authentication.AuthenticationServiceException; +import org.springframework.security.web.AuthenticationEntryPoint; + +import java.io.IOException; + +public class RestAuthenticationEntryPoint implements AuthenticationEntryPoint { + private final ObjectMapper mapper; + private final String challenge; + + public RestAuthenticationEntryPoint(ObjectMapper mapper, String challenge) { + this.mapper = mapper; + this.challenge = challenge; + } + + @Override + public void commence( + HttpServletRequest request, + HttpServletResponse response, + AuthenticationException exception + ) throws IOException { + String requestId = (String) request.getAttribute(RequestIdFilter.ATTRIBUTE); + boolean unavailable = exception instanceof AuthenticationServiceException; + response.setStatus(unavailable + ? HttpServletResponse.SC_SERVICE_UNAVAILABLE + : HttpServletResponse.SC_UNAUTHORIZED); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + response.setCharacterEncoding("UTF-8"); + if (!unavailable) { + response.setHeader("WWW-Authenticate", challenge); + } + mapper.writeValue(response.getOutputStream(), new ErrorResponse( + new ErrorResponse.ErrorBody( + unavailable ? "unavailable" : "unauthorized", + unavailable ? "Authentication service temporarily unavailable" : "Authentication required", + requestId))); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/security/SessionPrincipal.java b/src/main/java/com/example/mtusi_messenger/security/SessionPrincipal.java new file mode 100644 index 00000000..03c39516 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/security/SessionPrincipal.java @@ -0,0 +1,14 @@ +package com.example.mtusi_messenger.security; + +import com.example.mtusi_messenger.model.User; + +public record SessionPrincipal(User user, byte[] tokenHash) { + public SessionPrincipal { + tokenHash = tokenHash.clone(); + } + + @Override + public byte[] tokenHash() { + return tokenHash.clone(); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/AuthenticationService.java b/src/main/java/com/example/mtusi_messenger/service/AuthenticationService.java new file mode 100644 index 00000000..42abab99 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/AuthenticationService.java @@ -0,0 +1,104 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.cache.UserCache; +import com.example.mtusi_messenger.model.Session; +import com.example.mtusi_messenger.model.User; +import com.example.mtusi_messenger.repository.SessionRepository; +import com.example.mtusi_messenger.repository.UserRepository; +import com.example.mtusi_messenger.security.SessionPrincipal; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.dao.DataAccessException; +import org.springframework.security.authentication.AuthenticationServiceException; +import org.springframework.security.authentication.BadCredentialsException; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Base64; +import java.util.UUID; + +@Service +public class AuthenticationService { + private static final SecureRandom SECURE_RANDOM = new SecureRandom(); + + private final UserRepository users; + private final SessionRepository sessions; + private final UserCache userCache; + private final long ttlSeconds; + + public AuthenticationService( + UserRepository users, + SessionRepository sessions, + UserCache userCache, + @Value("${messenger.session.ttl-seconds:3600}") long ttlSeconds + ) { + if (ttlSeconds <= 0) { + throw new IllegalArgumentException("Session TTL must be positive"); + } + this.users = users; + this.sessions = sessions; + this.userCache = userCache; + this.ttlSeconds = ttlSeconds; + } + + public IssuedSession issueSession(User user) { + String token = newToken(); + Instant createdAt = Instant.now().truncatedTo(ChronoUnit.MICROS); + Instant expiresAt = createdAt.plusSeconds(ttlSeconds); + sessions.save(new Session( + UUID.randomUUID(), + user.id(), + hashToken(token), + createdAt, + expiresAt)); + return new IssuedSession(token, expiresAt, user); + } + + public SessionPrincipal authenticateBearer(String token) { + byte[] tokenHash = hashToken(token); + try { + UUID userId = sessions.findActiveUserId(tokenHash) + .orElseThrow(() -> new BadCredentialsException("Invalid or expired session")); + User user = userCache.find(userId).orElseGet(() -> { + User loaded = users.findById(userId) + .orElseThrow(() -> new BadCredentialsException("Invalid or expired session")); + userCache.put(loaded); + return loaded; + }); + return new SessionPrincipal(user, tokenHash); + } catch (DataAccessException e) { + throw new AuthenticationServiceException("Authentication database is unavailable", e); + } + } + + public void revoke(SessionPrincipal principal) { + try { + if (!sessions.revoke(principal.tokenHash())) { + throw new BadCredentialsException("Invalid or expired session"); + } + } catch (DataAccessException e) { + throw new AuthenticationServiceException("Authentication database is unavailable", e); + } + } + + private String newToken() { + byte[] bytes = new byte[32]; + SECURE_RANDOM.nextBytes(bytes); + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + + private byte[] hashToken(String token) { + try { + return MessageDigest.getInstance("SHA-256") + .digest(token.getBytes(StandardCharsets.US_ASCII)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + + public record IssuedSession(String token, Instant expiresAt, User user) {} +} diff --git a/src/main/java/com/example/mtusi_messenger/service/ChatService.java b/src/main/java/com/example/mtusi_messenger/service/ChatService.java new file mode 100644 index 00000000..38eeff2d --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/ChatService.java @@ -0,0 +1,38 @@ +package com.example.mtusi_messenger.service; + +import org.springframework.stereotype.Service; +import com.example.mtusi_messenger.dto.CreateChatRequest; +import com.example.mtusi_messenger.exception.ApiExceptions; +import com.example.mtusi_messenger.model.Chat; +import com.example.mtusi_messenger.repository.ChatRepository; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.*; + +@Service +public class ChatService { + private final ChatRepository chats; private final UserService users; private final CursorService cursors; + public ChatService(ChatRepository chats, UserService users, CursorService cursors) { this.chats=chats; this.users=users; this.cursors=cursors; } + public Chat create(CreateChatRequest r, UUID caller) { + Set ids = new LinkedHashSet<>(r.member_ids()); + if (ids.size() != r.member_ids().size() || !ids.contains(caller)) throw ApiExceptions.invalid("Invalid member_ids"); + for (UUID id : ids) if (id == null || !users.exists(id)) throw ApiExceptions.invalid("Invalid member_ids"); + return chats.save(new Chat(UUID.randomUUID(), r.title(), List.copyOf(ids), + Instant.now().truncatedTo(ChronoUnit.MICROS))); + } + public Chat getAccessible(UUID chatId, UUID userId) { + users.get(userId); + if (!chats.isMember(chatId, userId)) throw ApiExceptions.notFound("Chat not found"); + return chats.findById(chatId).orElseThrow(() -> ApiExceptions.notFound("Chat not found")); + } + public List page(UUID userId, int limit, String cursor) { + CursorService.Cursor c = cursor == null ? null : cursors.decode(cursor, "chats", userId); + return chats.findPageForUser(userId, c == null ? null : c.id(), c == null ? null : c.createdAt(), limit + 1); + } + public String nextCursor(UUID userId, + Chat last, + boolean hasNext + ) { + return hasNext ? cursors.encode("chats", userId, last.createdAt(), last.id()) : null; + } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/CursorService.java b/src/main/java/com/example/mtusi_messenger/service/CursorService.java new file mode 100644 index 00000000..87cd7ffb --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/CursorService.java @@ -0,0 +1,40 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.exception.ApiException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.stereotype.Service; +import com.example.mtusi_messenger.exception.ApiExceptions; +import java.time.Instant; +import java.util.Base64; +import java.util.UUID; + +@Service +public class CursorService { + private final ObjectMapper mapper; + public CursorService(ObjectMapper mapper) { this.mapper = mapper; } + + public String encode(String resource, UUID resourceId, Instant createdAt, UUID id) { + try { + byte[] json = mapper.writeValueAsBytes(new CursorPayload(1, resource, resourceId, createdAt.toString(), id)); + return Base64.getUrlEncoder().withoutPadding().encodeToString(json); + } catch (Exception e) { throw new IllegalStateException(e); } + } + public Cursor decode(String raw, String resource, UUID resourceId) { + if (raw == null || raw.isBlank()) throw ApiExceptions.invalid("Invalid cursor"); + try { + CursorPayload p = mapper.readValue(Base64.getUrlDecoder().decode(raw), CursorPayload.class); + if (p.v != 1 + || !resource.equals(p.resource) + || !Objects.equals(resourceId, p.resourceId) + || p.createdAt() == null + || p.createdAt().isBlank() + || p.id() == null + ) throw ApiExceptions.invalid("Invalid cursor"); + return new Cursor(Instant.parse(p.createdAt), p.id); + } catch (ApiException e) { throw e; } + catch (Exception e) { throw ApiExceptions.invalid("Invalid cursor"); } + } + public record Cursor(Instant createdAt, UUID id) {} + private record CursorPayload(int v, String resource, UUID resourceId, String createdAt, UUID id) {} + private static final class Objects { static boolean equals(Object a, Object b) { return java.util.Objects.equals(a,b); } } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/MessageService.java b/src/main/java/com/example/mtusi_messenger/service/MessageService.java new file mode 100644 index 00000000..a618ff46 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/MessageService.java @@ -0,0 +1,89 @@ +package com.example.mtusi_messenger.service; + +import com.github.f4b6a3.uuid.UuidCreator; +import com.example.mtusi_messenger.exception.ApiExceptions; +import com.example.mtusi_messenger.model.IdempotentMessage; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import com.example.mtusi_messenger.dto.CreateMessageRequest; +import com.example.mtusi_messenger.model.Message; +import com.example.mtusi_messenger.repository.MessageRepository; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +@Service +public class MessageService { + private static final String METHOD = "POST"; + private final MessageRepository messages; + private final ChatService chats; + private final CursorService cursors; + private final Duration idempotencyTtl; + + public MessageService( + MessageRepository messages, + ChatService chats, + CursorService cursors, + @Value("${messenger.idempotency.ttl-seconds:86400}") long idempotencyTtlSeconds + ) { + if (idempotencyTtlSeconds < 86400) { + throw new IllegalArgumentException("Idempotency TTL must be at least 24 hours"); + } + this.messages = messages; + this.chats = chats; + this.cursors = cursors; + this.idempotencyTtl = Duration.ofSeconds(idempotencyTtlSeconds); + } + + @Transactional + public Message create(UUID chatId, UUID senderId, CreateMessageRequest r, String idempotencyKey) { + chats.getAccessible(chatId, senderId); + String path = "/api/v1/chats/" + chatId + "/messages"; + byte[] requestHash = hash(r.text()); + Instant now = Instant.now().truncatedTo(ChronoUnit.MICROS); + + // The transaction-scoped PostgreSQL advisory lock serializes the same + // idempotency scope across every API replica. A hash collision only + // causes harmless extra serialization; the full scope is still checked. + messages.lockIdempotencyScope(senderId, METHOD, path, idempotencyKey); + messages.deleteExpiredIdempotency( + senderId, METHOD, path, idempotencyKey, now.minus(idempotencyTtl)); + + var existing = messages.findIdempotent(senderId, METHOD, path, idempotencyKey); + if (existing.isPresent()) { + IdempotentMessage value = existing.get(); + if (!Arrays.equals(value.requestHash(), requestHash)) { + throw ApiExceptions.conflict("Idempotency-Key was already used with another request body"); + } + return value.message(); + } + + Message message = messages.save(new Message( + UuidCreator.getTimeOrderedEpochPlus1(), chatId, senderId, r.text(), now)); + messages.saveIdempotency( + senderId, METHOD, path, idempotencyKey, requestHash, message.id(), now); + return message; + } + + private static byte[] hash(String text) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return digest.digest(text.getBytes(StandardCharsets.UTF_8)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + public List page(UUID chatId, UUID userId, int limit, String cursor) { + chats.getAccessible(chatId, userId); + CursorService.Cursor c = cursor == null ? null : cursors.decode(cursor, "messages", chatId); + return messages.findPage(chatId, c == null ? null : c.createdAt(), c == null ? null : c.id(), limit + 1); + } + public String nextCursor(UUID chatId, Message last, boolean hasNext) { return hasNext ? cursors.encode("messages", chatId, last.createdAt(), last.id()) : null; } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/UserService.java b/src/main/java/com/example/mtusi_messenger/service/UserService.java new file mode 100644 index 00000000..c8bed8b4 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/UserService.java @@ -0,0 +1,34 @@ +package com.example.mtusi_messenger.service; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.security.crypto.password.PasswordEncoder; +import com.example.mtusi_messenger.dto.CreateUserRequest; +import com.example.mtusi_messenger.exception.ApiExceptions; +import com.example.mtusi_messenger.model.User; +import com.example.mtusi_messenger.repository.UserRepository; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.UUID; + +@Service +@RequiredArgsConstructor +public class UserService { + private final UserRepository users; + private final PasswordEncoder passwordEncoder; + + public User create(CreateUserRequest r) { + String username = r.username(); String display = r.display_name(); + if (users.findByUsername(username).isPresent()) throw ApiExceptions.conflict("Username is already taken"); + String passwordHash = passwordEncoder.encode(r.password()); + try { + return users.save(new User(UUID.randomUUID(), username, display, + Instant.now().truncatedTo(ChronoUnit.MICROS)), passwordHash); + } catch (DuplicateKeyException e) { + throw ApiExceptions.conflict("Username is already taken"); + } + } + public User get(UUID id) { return users.findById(id).orElseThrow(() -> ApiExceptions.unauthorized("Unknown user identity")); } + public boolean exists(UUID id) { return users.findById(id).isPresent(); } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/UuidValidator.java b/src/main/java/com/example/mtusi_messenger/service/UuidValidator.java new file mode 100644 index 00000000..ea292fee --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/UuidValidator.java @@ -0,0 +1,27 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.exception.ApiExceptions; + +import java.util.UUID; +import java.util.regex.Pattern; + +public final class UuidValidator { + + private static final Pattern UUID_PATTERN = Pattern.compile( + "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" + ); + + private UuidValidator() {} + + public static UUID parse(String value) { + if (value == null || !UUID_PATTERN.matcher(value).matches()) { + throw ApiExceptions.invalid("Invalid UUID"); + } + + try { + return UUID.fromString(value); + } catch (IllegalArgumentException e) { + throw ApiExceptions.invalid("Invalid UUID"); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/example/mtusi_messenger/web/GlobalExceptionHandler.java b/src/main/java/com/example/mtusi_messenger/web/GlobalExceptionHandler.java new file mode 100644 index 00000000..2e77439e --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/GlobalExceptionHandler.java @@ -0,0 +1,48 @@ +package com.example.mtusi_messenger.web; + +import jakarta.servlet.http.HttpServletRequest; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.dao.DataAccessException; +import org.springframework.web.HttpMediaTypeNotSupportedException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; +import com.example.mtusi_messenger.dto.ErrorResponse; +import com.example.mtusi_messenger.exception.ApiException; + +@RestControllerAdvice +public class GlobalExceptionHandler { + @ExceptionHandler(ApiException.class) + ResponseEntity api(ApiException e, HttpServletRequest req) { + var body = body(e.code(), e.getMessage(), req); + var builder = ResponseEntity.status(e.status()); + if (e.status() == HttpStatus.UNAUTHORIZED && e.authenticate() != null) { + builder.header("WWW-Authenticate", e.authenticate()); + } + return builder.body(body); + } + @ExceptionHandler({MethodArgumentNotValidException.class, MethodArgumentTypeMismatchException.class, HttpMessageNotReadableException.class}) + ResponseEntity bad(Exception e, HttpServletRequest req) { return ResponseEntity.badRequest().body(body("invalid_request", "Invalid request", req)); } + @ExceptionHandler(HttpMediaTypeNotSupportedException.class) + ResponseEntity unsupported(HttpServletRequest req) { return ResponseEntity.status(415).body(body("unsupported_media_type", "Unsupported media type", req)); } + @ExceptionHandler(DataAccessException.class) + ResponseEntity unavailable(DataAccessException e, HttpServletRequest req) { + return ResponseEntity.status(503).body(body("unavailable", "Storage temporarily unavailable", req)); + } + @ExceptionHandler(Exception.class) + ResponseEntity internal(Exception e, HttpServletRequest req) { return ResponseEntity.status(500).body(body("internal_error", "Internal server error", req)); } + private ErrorResponse body( + String code, + String message, + HttpServletRequest req + ) { + String id = (String) req.getAttribute(RequestIdFilter.ATTRIBUTE); + + return new ErrorResponse( + new ErrorResponse.ErrorBody(code, message, id) + ); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/web/InstanceIdFilter.java b/src/main/java/com/example/mtusi_messenger/web/InstanceIdFilter.java new file mode 100644 index 00000000..9c56a393 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/InstanceIdFilter.java @@ -0,0 +1,47 @@ +package com.example.mtusi_messenger.web; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.UUID; + +@Component +@Order(Ordered.HIGHEST_PRECEDENCE + 1) +public class InstanceIdFilter extends OncePerRequestFilter { + public static final String HEADER = "X-Instance-Id"; + private final String instanceId; + + public InstanceIdFilter() { + String configured = System.getenv("INSTANCE_ID"); + this.instanceId = configured == null || configured.isBlank() + ? localHostOrRandom() + : configured; + } + + @Override + protected void doFilterInternal( + HttpServletRequest request, + HttpServletResponse response, + FilterChain filterChain + ) throws ServletException, IOException { + response.setHeader(HEADER, instanceId); + filterChain.doFilter(request, response); + } + + private static String localHostOrRandom() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + return "api-" + UUID.randomUUID(); + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/web/NotUnicodeBlank.java b/src/main/java/com/example/mtusi_messenger/web/NotUnicodeBlank.java new file mode 100644 index 00000000..a4213fc7 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/NotUnicodeBlank.java @@ -0,0 +1,21 @@ +package com.example.mtusi_messenger.web; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.*; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Documented +@Constraint(validatedBy = UnicodeBlankValidator.class) +@Target({ FIELD, PARAMETER, RECORD_COMPONENT, ANNOTATION_TYPE }) +@Retention(RUNTIME) +public @interface NotUnicodeBlank { + String message() default "must not be blank"; + Class[] groups() default {}; + Class[] payload() default {}; +} diff --git a/src/main/java/com/example/mtusi_messenger/web/RequestIdFilter.java b/src/main/java/com/example/mtusi_messenger/web/RequestIdFilter.java new file mode 100644 index 00000000..2b991b7a --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/RequestIdFilter.java @@ -0,0 +1,40 @@ +package com.example.mtusi_messenger.web; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.slf4j.MDC; +import org.springframework.stereotype.Component; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.web.filter.OncePerRequestFilter; +import java.io.IOException; +import java.util.UUID; + +@Component +@Order(Ordered.HIGHEST_PRECEDENCE) +public class RequestIdFilter extends OncePerRequestFilter { + + public static final String HEADER = "X-Request-Id"; + public static final String ATTRIBUTE = "request_id"; + + @Override + protected void doFilterInternal( + HttpServletRequest req, + HttpServletResponse res, + FilterChain chain + ) throws ServletException, IOException { + + String id = UUID.randomUUID().toString(); + + req.setAttribute(ATTRIBUTE, id); + res.setHeader(HEADER, id); + + try { + chain.doFilter(req, res); + } finally { + MDC.remove(ATTRIBUTE); + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/web/UnicodeBlankValidator.java b/src/main/java/com/example/mtusi_messenger/web/UnicodeBlankValidator.java new file mode 100644 index 00000000..59893614 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/UnicodeBlankValidator.java @@ -0,0 +1,23 @@ +package com.example.mtusi_messenger.web; + +import jakarta.validation.ConstraintValidator; +import jakarta.validation.ConstraintValidatorContext; + +public class UnicodeBlankValidator + implements ConstraintValidator { + + @Override + public boolean isValid( + String value, + ConstraintValidatorContext context) { + + if (value == null || value.isEmpty()) { + return false; + } + + return value.codePoints().anyMatch(codePoint -> + !Character.isWhitespace(codePoint) + && !Character.isSpaceChar(codePoint) + ); + } +} diff --git a/src/main/java/com/example/mtusi_messenger/web/UnicodeSize.java b/src/main/java/com/example/mtusi_messenger/web/UnicodeSize.java new file mode 100644 index 00000000..dde7ca9a --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/UnicodeSize.java @@ -0,0 +1,35 @@ +package com.example.mtusi_messenger.web; +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Documented +@Constraint(validatedBy = UnicodeSizeValidator.class) +@Target({ + FIELD, + METHOD, + PARAMETER, + ANNOTATION_TYPE +}) +@Retention(RUNTIME) +public @interface UnicodeSize { + + String message() default "size must be between {min} and {max} Unicode code points"; + + int min() default 0; + + int max() default Integer.MAX_VALUE; + + Class[] groups() default {}; + + Class[] payload() default {}; +} \ No newline at end of file diff --git a/src/main/java/com/example/mtusi_messenger/web/UnicodeSizeValidator.java b/src/main/java/com/example/mtusi_messenger/web/UnicodeSizeValidator.java new file mode 100644 index 00000000..f978d54d --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/UnicodeSizeValidator.java @@ -0,0 +1,32 @@ +package com.example.mtusi_messenger.web; + +import jakarta.validation.ConstraintValidator; +import jakarta.validation.ConstraintValidatorContext; + +public class UnicodeSizeValidator + implements ConstraintValidator { + + private int min; + private int max; + + @Override + public void initialize(UnicodeSize annotation) { + this.min = annotation.min(); + this.max = annotation.max(); + } + + @Override + public boolean isValid( + String value, + ConstraintValidatorContext context + ) { + if (value == null) { + return true; + } + + int codePointCount = value.codePointCount(0, value.length()); + + return codePointCount >= min + && codePointCount <= max; + } +} \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 00000000..0846c221 --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,62 @@ +spring: + application: + name: mtusi-messenger + + datasource: + url: ${DATABASE_URL:jdbc:postgresql://localhost:5432/messenger} + username: ${DATABASE_USER:messenger} + password: ${DATABASE_PASSWORD:messenger} + hikari: + maximum-pool-size: ${DATABASE_POOL_SIZE:10} + minimum-idle: ${DATABASE_MIN_IDLE:2} + connection-timeout: 5000 + validation-timeout: 2000 + idle-timeout: 600000 + max-lifetime: 1800000 + + data: + redis: + host: ${REDIS_HOST:localhost} + port: ${REDIS_PORT:6379} + connect-timeout: ${REDIS_CONNECT_TIMEOUT:1s} + timeout: ${REDIS_COMMAND_TIMEOUT:500ms} + lettuce: + pool: + max-active: ${REDIS_POOL_MAX_ACTIVE:8} + max-idle: ${REDIS_POOL_MAX_IDLE:8} + min-idle: ${REDIS_POOL_MIN_IDLE:0} + max-wait: ${REDIS_POOL_MAX_WAIT:500ms} + + jdbc: + template: + query-timeout: ${DATABASE_QUERY_TIMEOUT_SECONDS:5} + + transaction: + default-timeout: ${DATABASE_TRANSACTION_TIMEOUT_SECONDS:5} + + flyway: + # API replicas never race one another for migrations. Compose runs the + # same image once as the dedicated migrate service with this overridden. + enabled: ${FLYWAY_ENABLED:false} + locations: classpath:db/migration + + jackson: + deserialization: + fail-on-unknown-properties: true + + lifecycle: + timeout-per-shutdown-phase: 10s + +server: + address: ${APP_HOST:0.0.0.0} + port: ${APP_PORT:8080} + shutdown: graceful + +messenger: + storage: postgres + session: + ttl-seconds: ${SESSION_TTL_SECONDS:3600} + cache: + user-ttl-seconds: ${USER_CACHE_TTL_SECONDS:300} + idempotency: + ttl-seconds: ${IDEMPOTENCY_TTL_SECONDS:86400} diff --git a/src/main/resources/db/migration/V1__create_messenger_schema.sql b/src/main/resources/db/migration/V1__create_messenger_schema.sql new file mode 100644 index 00000000..06885a1c --- /dev/null +++ b/src/main/resources/db/migration/V1__create_messenger_schema.sql @@ -0,0 +1,60 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL, + display_name text NOT NULL, + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT users_username_uk UNIQUE (username), + CONSTRAINT users_username_format_chk + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_length_chk + CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT chats_title_length_chk + CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL, + user_id uuid NOT NULL, + joined_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT chat_members_pk PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_chat_fk + FOREIGN KEY (chat_id) REFERENCES chats(id) ON DELETE CASCADE, + CONSTRAINT chat_members_user_fk + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE RESTRICT +); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT messages_text_length_chk + CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT messages_sender_membership_fk + FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members(chat_id, user_id) + ON DELETE RESTRICT +); + +CREATE INDEX chat_members_user_chat_idx + ON chat_members (user_id, chat_id); + +CREATE INDEX chats_created_id_idx + ON chats (created_at, id); + +CREATE INDEX messages_chat_created_id_idx + ON messages (chat_id, created_at, id); + +CREATE INDEX messages_sender_created_idx + ON messages (sender_id, created_at); diff --git a/src/main/resources/db/migration/V2__add_credentials_and_sessions.sql b/src/main/resources/db/migration/V2__add_credentials_and_sessions.sql new file mode 100644 index 00000000..d5d74279 --- /dev/null +++ b/src/main/resources/db/migration/V2__add_credentials_and_sessions.sql @@ -0,0 +1,28 @@ +ALTER TABLE users + ADD COLUMN password_hash text; + +ALTER TABLE users + ALTER COLUMN password_hash SET NOT NULL; + +CREATE TABLE sessions ( + id uuid PRIMARY KEY, + user_id uuid NOT NULL, + token_hash bytea NOT NULL, + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + + CONSTRAINT sessions_user_fk + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + CONSTRAINT sessions_token_hash_uk UNIQUE (token_hash), + CONSTRAINT sessions_token_hash_length_chk + CHECK (octet_length(token_hash) = 32), + CONSTRAINT sessions_expiry_chk + CHECK (expires_at > created_at), + CONSTRAINT sessions_revocation_chk + CHECK (revoked_at IS NULL OR revoked_at >= created_at) +); + +CREATE INDEX sessions_active_expiry_idx + ON sessions (expires_at) + WHERE revoked_at IS NULL; diff --git a/src/main/resources/db/migration/V3__add_message_idempotency.sql b/src/main/resources/db/migration/V3__add_message_idempotency.sql new file mode 100644 index 00000000..dfa127ca --- /dev/null +++ b/src/main/resources/db/migration/V3__add_message_idempotency.sql @@ -0,0 +1,25 @@ +CREATE TABLE message_idempotency ( + user_id uuid NOT NULL, + method varchar(8) NOT NULL, + request_path text NOT NULL, + idempotency_key varchar(128) NOT NULL, + request_hash bytea NOT NULL, + message_id uuid NOT NULL, + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT message_idempotency_pk + PRIMARY KEY (user_id, method, request_path, idempotency_key), + CONSTRAINT message_idempotency_user_fk + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + CONSTRAINT message_idempotency_message_fk + FOREIGN KEY (message_id) REFERENCES messages(id) ON DELETE CASCADE, + CONSTRAINT message_idempotency_method_chk + CHECK (method = 'POST'), + CONSTRAINT message_idempotency_key_format_chk + CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT message_idempotency_hash_length_chk + CHECK (octet_length(request_hash) = 32) +); + +CREATE INDEX message_idempotency_created_idx + ON message_idempotency (created_at); diff --git a/src/test/java/com/example/mtusi_messenger/MtusiMessengerApplicationTests.java b/src/test/java/com/example/mtusi_messenger/MtusiMessengerApplicationTests.java new file mode 100644 index 00000000..6a71efe7 --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/MtusiMessengerApplicationTests.java @@ -0,0 +1,368 @@ +package com.example.mtusi_messenger; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +@SpringBootTest +@AutoConfigureMockMvc +class MtusiMessengerApplicationTests { + + private static final String USERS_URL = "/api/v1/users"; + private static final String CHATS_URL = "/api/v1/chats"; + private static final String SESSIONS_URL = "/api/v1/auth/sessions"; + private static final String TEST_PASSWORD = "correct horse battery staple"; + + @Autowired + private MockMvc mockMvc; + + @Autowired + private ObjectMapper objectMapper; + + private User alice; + private User bob; + private User eve; + + @BeforeEach + void setUp() throws Exception { + String suffix = UUID.randomUUID() + .toString() + .replace("-", "") + .substring(0, 12); + + alice = registerUser("alice_" + suffix, "Alice"); + bob = registerUser("bob_" + suffix, "Bob"); + eve = registerUser("eve_" + suffix, "Eve"); + } + + @Test + void shouldReturn400WhenJsonIsMalformed() throws Exception { + mockMvc.perform(post(USERS_URL) + .contentType(MediaType.APPLICATION_JSON) + .content("{ \"username\": ")) + .andExpect(status().isBadRequest()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("invalid_request")); + } + + @Test + void shouldReturn400WhenPasswordContainsOnlyWhitespace() throws Exception { + String requestBody = objectMapper.writeValueAsString( + Map.of( + "username", "blank_password_" + UUID.randomUUID().toString().substring(0, 8), + "display_name", "Blank password probe", + "password", " " + ) + ); + + mockMvc.perform(post(USERS_URL) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isBadRequest()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("invalid_request")); + } + + @Test + void shouldNotAllowNonMemberToReadChatMessages() throws Exception { + String chatId = createChat(alice, bob); + + mockMvc.perform(get(CHATS_URL + "/" + chatId + "/messages") + .header("Authorization", bearer(eve)) + .param("limit", "10")) + .andExpect(status().isNotFound()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("not_found")); + } + + @Test + void shouldNotAllowNonMemberToSendMessage() throws Exception { + String chatId = createChat(alice, bob); + + String requestBody = """ + { + "text": "Hello from Eve" + } + """; + + mockMvc.perform(post(CHATS_URL + "/" + chatId + "/messages") + .header("Authorization", bearer(eve)) + .header("Idempotency-Key", UUID.randomUUID().toString()) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isNotFound()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("not_found")); + } + + @Test + void shouldReturn400WhenMessageTextIsEmpty() throws Exception { + String chatId = createChat(alice, bob); + + String requestBody = """ + { + "text": "" + } + """; + + mockMvc.perform(post(CHATS_URL + "/" + chatId + "/messages") + .header("Authorization", bearer(alice)) + .header("Idempotency-Key", UUID.randomUUID().toString()) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isBadRequest()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("invalid_request")); + } + + @Test + void shouldReturn400WhenMessageTextIsMissing() throws Exception { + String chatId = createChat(alice, bob); + + mockMvc.perform(post(CHATS_URL + "/" + chatId + "/messages") + .header("Authorization", bearer(alice)) + .header("Idempotency-Key", UUID.randomUUID().toString()) + .contentType(MediaType.APPLICATION_JSON) + .content("{}")) + .andExpect(status().isBadRequest()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("invalid_request")); + } + + @Test + void shouldReturn409WhenUsernameAlreadyExists() throws Exception { + String requestBody = """ + { + "username": "%s", + "display_name": "Another Alice", + "password": "correct horse battery staple" + } + """.formatted(alice.username()); + + mockMvc.perform(post(USERS_URL) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isConflict()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.error.code") + .value("conflict")); + } + + @Test + void shouldReturnAllMessagesAcrossMultiplePagesWithoutDuplicates() + throws Exception { + + String chatId = createChat(alice, bob); + + int messageCount = 5; + int pageSize = 2; + int expectedPageCount = 3; + + List expectedTexts = new ArrayList<>(); + + for (int i = 1; i <= messageCount; i++) { + String text = "Message " + i; + sendMessage(chatId, alice, text); + expectedTexts.add(text); + } + + List actualIds = new ArrayList<>(); + List actualTexts = new ArrayList<>(); + + String cursor = null; + int pageCount = 0; + + while (true) { + var request = get(CHATS_URL + "/" + chatId + "/messages") + .header("Authorization", bearer(alice)) + .param("limit", String.valueOf(pageSize)); + + if (cursor != null) { + request.param("cursor", cursor); + } + + MvcResult result = mockMvc.perform(request) + .andExpect(status().isOk()) + .andExpect(header().exists("X-Request-Id")) + .andExpect(jsonPath("$.items").isArray()) + .andReturn(); + + JsonNode response = objectMapper.readTree( + result.getResponse().getContentAsString() + ); + + JsonNode items = response.get("items"); + + assertNotNull(items); + assertTrue(items.size() <= pageSize); + + for (JsonNode item : items) { + actualIds.add(item.get("id").asText()); + actualTexts.add(item.get("text").asText()); + } + + pageCount++; + + JsonNode nextCursor = response.get("next_cursor"); + + if (nextCursor.isNull()) { + break; + } + + cursor = nextCursor.asText(); + + assertFalse(cursor.isBlank(), + "next_cursor must not be blank when more pages exist"); + + assertTrue(pageCount <= messageCount, + "Pagination did not terminate"); + } + + assertEquals(expectedPageCount, pageCount); + + assertEquals(messageCount, actualIds.size()); + assertEquals(messageCount, actualTexts.size()); + + assertEquals(actualIds.size(), new HashSet<>(actualIds).size()); + + assertEquals(expectedTexts, actualTexts); + } + + @Test + void shouldIgnoreXUserIdForProtectedEndpoints() throws Exception { + mockMvc.perform(get(USERS_URL + "/me") + .header("X-User-Id", alice.id())) + .andExpect(status().isUnauthorized()) + .andExpect(header().string("WWW-Authenticate", "Bearer realm=\"messenger\"")); + } + + @Test + void shouldRevokeCurrentSession() throws Exception { + mockMvc.perform(delete(SESSIONS_URL + "/current") + .header("Authorization", bearer(alice))) + .andExpect(status().isNoContent()); + + mockMvc.perform(get(USERS_URL + "/me") + .header("Authorization", bearer(alice))) + .andExpect(status().isUnauthorized()) + .andExpect(header().string("WWW-Authenticate", "Bearer realm=\"messenger\"")); + } + + @Test + void shouldRejectWrongPasswordWithBasicChallenge() throws Exception { + String basic = Base64.getEncoder().encodeToString( + (alice.username() + ":definitely-wrong-password") + .getBytes(java.nio.charset.StandardCharsets.UTF_8)); + + mockMvc.perform(post(SESSIONS_URL) + .header("Authorization", "Basic " + basic)) + .andExpect(status().isUnauthorized()) + .andExpect(header().string( + "WWW-Authenticate", + "Basic realm=\"messenger\", charset=\"UTF-8\"")); + } + + private User registerUser(String username, String displayName) + throws Exception { + + String requestBody = objectMapper.writeValueAsString( + Map.of( + "username", username, + "display_name", displayName, + "password", TEST_PASSWORD + ) + ); + + MvcResult result = mockMvc.perform(post(USERS_URL) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isCreated()) + .andExpect(header().exists("X-Request-Id")) + .andReturn(); + + JsonNode response = objectMapper.readTree( + result.getResponse().getContentAsString() + ); + + String basic = Base64.getEncoder().encodeToString( + (username + ":" + TEST_PASSWORD).getBytes(java.nio.charset.StandardCharsets.UTF_8)); + MvcResult loginResult = mockMvc.perform(post(SESSIONS_URL) + .header("Authorization", "Basic " + basic)) + .andExpect(status().isCreated()) + .andReturn(); + JsonNode session = objectMapper.readTree(loginResult.getResponse().getContentAsString()); + + return new User( + response.get("id").asText(), + response.get("username").asText(), + session.get("token").asText() + ); + } + + private String createChat(User owner, User member) + throws Exception { + + String requestBody = objectMapper.writeValueAsString( + java.util.Map.of( + "title", "Test chat " + UUID.randomUUID(), + "member_ids", List.of(owner.id(), member.id()) + ) + ); + + MvcResult result = mockMvc.perform(post(CHATS_URL) + .header("Authorization", bearer(owner)) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isCreated()) + .andExpect(header().exists("X-Request-Id")) + .andReturn(); + + JsonNode response = objectMapper.readTree( + result.getResponse().getContentAsString() + ); + + return response.get("id").asText(); + } + + private void sendMessage(String chatId, User sender, String text) + throws Exception { + + String requestBody = objectMapper.writeValueAsString( + java.util.Map.of("text", text) + ); + + mockMvc.perform(post(CHATS_URL + "/" + chatId + "/messages") + .header("Authorization", bearer(sender)) + .header("Idempotency-Key", UUID.randomUUID().toString()) + .contentType(MediaType.APPLICATION_JSON) + .content(requestBody)) + .andExpect(status().isCreated()) + .andExpect(header().exists("X-Request-Id")); + } + + private static String bearer(User user) { + return "Bearer " + user.token(); + } + + private record User(String id, String username, String token) { + } +} diff --git a/src/test/java/com/example/mtusi_messenger/cache/UserCacheTest.java b/src/test/java/com/example/mtusi_messenger/cache/UserCacheTest.java new file mode 100644 index 00000000..fdc05347 --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/cache/UserCacheTest.java @@ -0,0 +1,31 @@ +package com.example.mtusi_messenger.cache; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.ValueOperations; + +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class UserCacheTest { + + @Test + void shouldEvictJsonNullAndReturnCacheMiss() { + UUID userId = UUID.randomUUID(); + String key = "messenger:user:" + userId; + StringRedisTemplate redis = mock(StringRedisTemplate.class); + @SuppressWarnings("unchecked") + ValueOperations values = mock(ValueOperations.class); + when(redis.opsForValue()).thenReturn(values); + when(values.get(key)).thenReturn("null"); + UserCache cache = new UserCache(redis, new ObjectMapper(), 300); + + assertTrue(cache.find(userId).isEmpty()); + verify(redis).delete(key); + } +} diff --git a/student_tests/README.md b/student_tests/README.md new file mode 100644 index 00000000..0ac05425 --- /dev/null +++ b/student_tests/README.md @@ -0,0 +1,11 @@ +# Student-owned tests + +This directory contains additional experiments written for this implementation. +The course-provided black-box suite remains in `tests/`; Spring tests remain in +`src/test/java`. + +- `idempotency_concurrent.py` — one key through multiple API replicas. +- `failover_each.py` — sequential process failover of both Compose replicas. +- `rolling_update.py` — sequential replacement under write load. +- `node_failover.py` — hard loss of one worker in the disposable three-node + Swarm lab, task relocation and confirmed-data verification. diff --git a/student_tests/failover_each.py b/student_tests/failover_each.py new file mode 100644 index 00000000..996f072d --- /dev/null +++ b/student_tests/failover_each.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Stop every observed API replica in turn and record recovery metrics.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + + +def executable(path, label): + value = Path(path).expanduser().resolve() + check(value.is_file() and os.access(value, os.X_OK), label + ' must be executable') + return value + + +def observe_instances(client, actor, attempts=60): + result = set() + for _ in range(attempts): + result.add(client.call( + 'GET', '/api/v1/users/me', actor=actor, schema='User' + )['headers']['X-Instance-Id']) + return result + + +def message_history(client, actor, chat_id): + path = '/api/v1/chats/' + chat_id + '/messages?limit=100' + result = [] + for _ in range(1000): + page = client.call( + 'GET', path, actor=actor, schema='MessagePage')['json'] + result.extend(page['items']) + if page['next_cursor'] is None: + return result + path = ('/api/v1/chats/' + chat_id + '/messages?limit=100&cursor=' + + quote(page['next_cursor'], safe='')) + raise Failure('history pagination exceeded safety bound') + + +def main(): + def configure(parser): + parser.add_argument('--action-script', required=True) + parser.add_argument('--restore-script', required=True) + parser.add_argument('--window', type=float, default=15) + parser.add_argument('--output', default=None) + args = arguments('Sequential failover of every API replica', configure) + check(5 <= args.window <= 60, 'window must be 5..60 seconds') + action = executable(args.action_script, 'action script') + restore = executable(args.restore_script, 'restore script') + client = Client(args.base_url, args.ca_file, timeout=2) + alice, bob, _, chat = client.bootstrap() + targets = sorted(observe_instances(client, alice)) + check(len(targets) >= 2, 'expected at least two API replicas') + targets = targets[:2] + summaries = [] + + for target in targets: + check(target in observe_instances(client, alice), + 'target replica is not observable before stop') + env = dict(os.environ) + env['TARGET_INSTANCE_ID'] = target + process = subprocess.Popen( + [str(action)], env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + started = time.monotonic() + first_recovery = None + reads = writes = failures = 0 + confirmed = [] + key = uuid.uuid4().hex + text = 'failover-each-' + target + '-' + key + + while time.monotonic() - started < args.window: + try: + if (reads + writes) % 2 == 0: + response = client.message(alice, chat, text, key=key) + confirmed.append(response['json']) + writes += 1 + key = uuid.uuid4().hex + text = 'failover-each-' + target + '-' + key + else: + client.call('GET', '/api/v1/users/me', actor=bob, schema='User') + reads += 1 + if process.poll() == 0 and first_recovery is None: + first_recovery = time.monotonic() - started + except Failure: + failures += 1 + time.sleep(0.05) + + check(process.wait(timeout=5) == 0, 'stop-one action failed') + elapsed = time.monotonic() - started + check(first_recovery is not None and first_recovery <= 10, + 'recovery exceeded 10 seconds') + history = message_history(client, bob, chat['id']) + history_ids = {message['id'] for message in history} + check(all(message['id'] in history_ids for message in confirmed), + 'confirmed message disappeared after failover') + + summaries.append({ + 'target_instance_id': target, + 'window_seconds': round(elapsed, 3), + 'successful_reads': reads, + 'successful_writes': writes, + 'failed_attempts': failures, + 'successful_rps': round((reads + writes) / elapsed, 3), + 'first_recovery_seconds': round(first_recovery, 3), + }) + + check(subprocess.run( + [str(restore)], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=100).returncode == 0, + 'restore action failed') + check(target in observe_instances(client, alice), + 'stopped replica did not return after restore') + + report = { + 'scenario': 'sequential failover of each API replica', + 'replicas': len(targets), + 'results': summaries, + } + output = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(output + '\n', encoding='utf-8') + print(output) + + +if __name__ == '__main__': + run(main) diff --git a/student_tests/idempotency_concurrent.py b/student_tests/idempotency_concurrent.py new file mode 100644 index 00000000..1ed1f04c --- /dev/null +++ b/student_tests/idempotency_concurrent.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Prove that one idempotency scope is shared across API replicas.""" +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import sys +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, arguments, check, run + + +def main(): + args = arguments('Concurrent cross-replica idempotency check') + bootstrap = Client(args.base_url, args.ca_file, timeout=5) + alice, _, _, chat = bootstrap.bootstrap() + key = uuid.uuid4().hex + text = 'Один результат для всех реплик ' + key + path = '/api/v1/chats/' + chat['id'] + '/messages' + + def send(_): + client = Client(args.base_url, args.ca_file, timeout=10) + return client.call( + 'POST', + path, + 201, + actor=alice, + body={'text': text}, + headers={'Idempotency-Key': key}, + schema='Message') + + with ThreadPoolExecutor(max_workers=16) as pool: + responses = list(pool.map(send, range(32))) + + messages = [response['json'] for response in responses] + instances = { + response['headers']['X-Instance-Id'] for response in responses + } + check(all(message == messages[0] for message in messages), + 'same key returned different messages') + check(len(instances) >= 2, + 'requests did not exercise at least two API replicas') + + page = bootstrap.call( + 'GET', path + '?limit=100', actor=alice, schema='MessagePage')['json'] + matches = [item for item in page['items'] if item['id'] == messages[0]['id']] + check(len(matches) == 1, 'idempotent message is missing or duplicated') + print('PASS: 32 concurrent requests used ' + str(len(instances)) + + ' replicas and created one message') + + +if __name__ == '__main__': + run(main) diff --git a/student_tests/node_failover.py b/student_tests/node_failover.py new file mode 100644 index 00000000..2cf00a6d --- /dev/null +++ b/student_tests/node_failover.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Hard-stop one Swarm worker and prove API task relocation and data safety.""" +import json +import atexit +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import subprocess +import sys +import time +import uuid +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + + +def command(*parts, timeout=30): + result = subprocess.run( + parts, capture_output=True, text=True, timeout=timeout) + check(result.returncode == 0, 'command failed: ' + ' '.join(parts[:3])) + return result.stdout.strip() + + +def api_tasks(manager): + output = command( + 'docker', 'exec', manager, 'docker', 'service', 'ps', 'hla-app_api', + '--filter', 'desired-state=running', + '--format', '{{.ID}}\t{{.Node}}\t{{.CurrentState}}') + result = [] + for line in output.splitlines(): + fields = line.split('\t') + if len(fields) == 3 and fields[2].startswith('Running'): + result.append({'task_id': fields[0], 'node': fields[1]}) + return result + + +def history(client, actor, chat_id): + path = '/api/v1/chats/' + chat_id + '/messages?limit=100' + result = [] + for _ in range(1000): + page = client.call('GET', path, actor=actor, schema='MessagePage')['json'] + result.extend(page['items']) + if page['next_cursor'] is None: + return result + path = ('/api/v1/chats/' + chat_id + '/messages?limit=100&cursor=' + + quote(page['next_cursor'], safe='')) + raise Failure('history pagination exceeded safety bound') + + +def observe(client, actor, attempts=40): + def request(_): + local = Client(client.base, timeout=2) + return local.call( + 'GET', '/api/v1/users/me', actor=actor, schema='User') + with ThreadPoolExecutor(max_workers=16) as pool: + responses = list(pool.map(request, range(attempts))) + return {response['headers']['X-Instance-Id'] for response in responses} + + +def restore_worker(manager, node): + subprocess.run( + ['docker', 'start', node], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=30) + deadline = time.monotonic() + 90 + status = '' + while time.monotonic() < deadline: + result = subprocess.run( + ['docker', 'exec', manager, 'docker', 'node', 'ls', + '--filter', 'name=' + node, '--format', '{{.Status}}'], + capture_output=True, text=True, timeout=30) + status = result.stdout.strip() + if result.returncode == 0 and status == 'Ready': + return True + time.sleep(2) + return False + + +def main(): + def configure(parser): + parser.add_argument('--manager-container', default='hla-swarm-manager') + parser.add_argument('--window', type=float, default=30) + parser.add_argument('--output', default=None) + args = arguments('Swarm worker loss and API relocation', configure) + check(15 <= args.window <= 120, 'window must be 15..120 seconds') + client = Client(args.base_url, args.ca_file, timeout=2) + alice, bob, _, chat = client.bootstrap() + before_instances = observe(client, alice) + check(len(before_instances) >= 2, 'expected two API instances before node loss') + before_tasks = api_tasks(args.manager_container) + check(len(before_tasks) == 2, 'expected two running API tasks') + candidates = [task for task in before_tasks + if task['node'].startswith('hla-swarm-worker')] + check(bool(candidates), 'no API task is running on a worker node') + victim = candidates[0] + confirmed_before = client.message( + alice, chat, 'confirmed-before-node-loss-' + uuid.uuid4().hex)['json'] + + started = time.monotonic() + command('docker', 'stop', '--time', '1', victim['node'], timeout=15) + restore_at_exit = lambda: restore_worker(args.manager_container, victim['node']) + atexit.register(restore_at_exit) + first_recovery = None + reads = writes = failures = 0 + confirmed_during = [] + key = uuid.uuid4().hex + text = 'during-node-loss-' + key + while time.monotonic() - started < args.window: + try: + if (reads + writes) % 2 == 0: + response = client.message(alice, chat, text, key=key) + confirmed_during.append(response['json']) + writes += 1 + key = uuid.uuid4().hex + text = 'during-node-loss-' + key + else: + client.call('GET', '/api/v1/users/me', actor=bob, schema='User') + reads += 1 + if first_recovery is None: + first_recovery = time.monotonic() - started + except Failure: + failures += 1 + time.sleep(0.05) + + deadline = time.monotonic() + 90 + after_tasks = [] + while time.monotonic() < deadline: + after_tasks = api_tasks(args.manager_container) + if (len(after_tasks) == 2 + and all(task['node'] != victim['node'] for task in after_tasks) + and any(task['task_id'] != victim['task_id'] for task in after_tasks)): + break + time.sleep(2) + check(len(after_tasks) == 2, 'API service did not return to two tasks') + check(all(task['node'] != victim['node'] for task in after_tasks), + 'API task was not relocated away from failed node') + check(first_recovery is not None and first_recovery <= 10, + 'API did not recover within 10 seconds') + + stored = history(client, bob, chat['id']) + stored_ids = {message['id'] for message in stored} + check(confirmed_before['id'] in stored_ids, + 'pre-failure confirmed message was lost') + check(all(message['id'] in stored_ids for message in confirmed_during), + 'message confirmed during node loss was lost') + check(restore_worker(args.manager_container, victim['node']), + 'failed worker did not rejoin the Swarm') + atexit.unregister(restore_at_exit) + after_instances = observe(client, alice, attempts=96) + check(len(after_instances - before_instances) >= 1, + 'replacement API process was not visible through X-Instance-Id') + + elapsed = time.monotonic() - started + report = { + 'scenario': 'hard loss of one Swarm worker', + 'failed_node': victim['node'], + 'old_task_id': victim['task_id'], + 'tasks_before': before_tasks, + 'tasks_after_relocation': after_tasks, + 'instance_ids_before': sorted(before_instances), + 'instance_ids_after': sorted(after_instances), + 'successful_reads': reads, + 'successful_writes': writes, + 'failed_attempts': failures, + 'first_recovery_seconds': round(first_recovery, 3), + 'successful_rps': round((reads + writes) / args.window, 3), + 'experiment_seconds_including_rejoin': round(elapsed, 3), + 'confirmed_data_preserved': True, + 'database_node_failed': False, + 'note': ('PostgreSQL, Nginx, manager quorum and local volumes are not ' + 'protected by this experiment.'), + } + output = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(output + '\n', encoding='utf-8') + print(output) + + +if __name__ == '__main__': + run(main) diff --git a/student_tests/rolling_update.py b/student_tests/rolling_update.py new file mode 100644 index 00000000..20effca9 --- /dev/null +++ b/student_tests/rolling_update.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Run closed-loop writes while API replicas are replaced one at a time.""" +from concurrent.futures import ThreadPoolExecutor +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import threading +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + + +def main(): + def configure(parser): + parser.add_argument('--action-script', required=True) + parser.add_argument('--duration', type=float, default=180) + parser.add_argument('--concurrency', type=int, default=4) + parser.add_argument('--output', default=None) + args = arguments('Rolling replacement under write load', configure) + check(30 <= args.duration <= 300, 'duration must be 30..300 seconds') + check(1 <= args.concurrency <= 32, 'concurrency must be 1..32') + action = Path(args.action_script).expanduser().resolve() + check(action.is_file() and os.access(action, os.X_OK), + 'action script must be executable') + bootstrap = Client(args.base_url, args.ca_file, timeout=5) + alice, _, _, chat = bootstrap.bootstrap() + barrier = threading.Barrier(args.concurrency + 1) + + def worker(worker_id): + client = Client(args.base_url, args.ca_file, timeout=5) + barrier.wait() + deadline = time.monotonic() + args.duration + latencies = [] + errors = 0 + instances = set() + index = 0 + while time.monotonic() < deadline: + started = time.monotonic() + try: + response = client.message( + alice, chat, 'rolling-%s-%s' % (worker_id, index)) + latencies.append(time.monotonic() - started) + instances.add(response['headers']['X-Instance-Id']) + except Failure: + errors += 1 + index += 1 + return latencies, errors, instances + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + futures = [pool.submit(worker, number) + for number in range(args.concurrency)] + barrier.wait() + time.sleep(3) + action_started = time.monotonic() + process = subprocess.run( + [str(action)], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=args.duration - 5) + action_seconds = time.monotonic() - action_started + check(process.returncode == 0, 'rolling action failed') + results = [future.result() for future in futures] + + elapsed = time.monotonic() - started + latencies = sorted(value for values, _, _ in results for value in values) + errors = sum(value for _, value, _ in results) + instances = set().union(*(value for _, _, value in results)) + total = len(latencies) + errors + + def percentile(fraction): + if not latencies: + return None + index = max(0, math.ceil(fraction * len(latencies)) - 1) + return round(1000 * latencies[index], 3) + + report = { + 'scenario': 'sequential API replacement under closed-loop writes', + 'requested_duration_seconds': args.duration, + 'elapsed_seconds': round(elapsed, 3), + 'rolling_action_seconds': round(action_seconds, 3), + 'concurrency': args.concurrency, + 'observed_instance_ids': sorted(instances), + 'successful_requests': len(latencies), + 'failed_requests': errors, + 'error_ratio': round(errors / total, 6) if total else None, + 'successful_rps': round(len(latencies) / elapsed, 3), + 'successful_latency_ms': { + 'p50': percentile(.50), + 'p95': percentile(.95), + 'p99': percentile(.99), + }, + 'note': 'Closed-loop result on one host; success latency excludes failures.', + } + output = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(output + '\n', encoding='utf-8') + print(output) + check(bool(latencies), 'no successful writes') + check(len(instances) >= 3, + 'rolling replacement did not expose replacement instance IDs') + + +if __name__ == '__main__': + run(main) diff --git a/swarm/nginx.Dockerfile b/swarm/nginx.Dockerfile new file mode 100644 index 00000000..9d74ead5 --- /dev/null +++ b/swarm/nginx.Dockerfile @@ -0,0 +1,3 @@ +FROM nginx:1.27.4-bookworm +COPY swarm/nginx.conf /etc/nginx/nginx.conf +HEALTHCHECK --interval=2s --timeout=2s --retries=15 CMD nginx -t diff --git a/swarm/nginx.conf b/swarm/nginx.conf new file mode 100644 index 00000000..edd794c9 --- /dev/null +++ b/swarm/nginx.conf @@ -0,0 +1,59 @@ +worker_processes auto; +worker_shutdown_timeout 10s; + +events { + worker_connections 1024; +} + +http { + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log warn; + resolver 127.0.0.11 valid=2s ipv6=off; + resolver_timeout 1s; + + upstream api_backend { + zone api_backend 64k; + least_conn; + server tasks.hla-app_api:8080 resolve max_fails=1 fail_timeout=2s; + keepalive 32; + } + + server { + listen 80; + server_name _; + + location = /nginx-health { + default_type application/json; + return 200 '{"status":"ok"}'; + } + + location ~ ^/api/v1/chats/[0-9a-fA-F-]+/messages$ { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_request_buffering on; + proxy_connect_timeout 500ms; + proxy_send_timeout 4s; + proxy_read_timeout 4s; + proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent; + proxy_next_upstream_tries 2; + } + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 500ms; + proxy_send_timeout 4s; + proxy_read_timeout 4s; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + } + } +} diff --git a/swarm/stack.app.yaml b/swarm/stack.app.yaml new file mode 100644 index 00000000..d97b083a --- /dev/null +++ b/swarm/stack.app.yaml @@ -0,0 +1,75 @@ +version: "3.9" + +services: + api: + image: hla-swarm-registry:5000/hla-lab3-api:swarm + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL: jdbc:postgresql://hla-state_postgres:5432/${DATABASE_NAME:-messenger} + DATABASE_USER: ${DATABASE_USER:-messenger} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD is required} + DATABASE_POOL_SIZE: 10 + DATABASE_MIN_IDLE: 2 + DATABASE_QUERY_TIMEOUT_SECONDS: 5 + DATABASE_TRANSACTION_TIMEOUT_SECONDS: 5 + REDIS_HOST: hla-state_redis + REDIS_PORT: 6379 + REDIS_CONNECT_TIMEOUT: 1s + REDIS_COMMAND_TIMEOUT: 500ms + SESSION_TTL_SECONDS: 3600 + USER_CACHE_TTL_SECONDS: 300 + IDEMPOTENCY_TTL_SECONDS: 86400 + FLYWAY_ENABLED: "false" + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://localhost:8080/health/ready"] + interval: 2s + timeout: 2s + retries: 20 + start_period: 45s + networks: [backend] + deploy: + replicas: 2 + endpoint_mode: dnsrr + placement: + max_replicas_per_node: 1 + restart_policy: + condition: on-failure + delay: 2s + update_config: + parallelism: 1 + order: start-first + monitor: 15s + failure_action: rollback + rollback_config: + parallelism: 1 + order: stop-first + resources: + limits: + cpus: "0.75" + memory: 768M + + nginx: + image: hla-swarm-registry:5000/hla-lab3-nginx:swarm + ports: + - target: 80 + published: 8080 + protocol: tcp + mode: ingress + networks: [backend] + deploy: + replicas: 1 + placement: + constraints: + - node.labels.hla.lb == true + restart_policy: + condition: on-failure + resources: + limits: + cpus: "0.25" + memory: 128M + +networks: + backend: + external: true + name: hla-state_backend diff --git a/swarm/stack.state.yaml b/swarm/stack.state.yaml new file mode 100644 index 00000000..9abb7981 --- /dev/null +++ b/swarm/stack.state.yaml @@ -0,0 +1,60 @@ +version: "3.9" + +services: + postgres: + image: postgres:16.4-bookworm + environment: + POSTGRES_DB: ${DATABASE_NAME:-messenger} + POSTGRES_USER: ${DATABASE_USER:-messenger} + POSTGRES_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD is required} + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 2s + timeout: 2s + retries: 30 + networks: [backend] + deploy: + replicas: 1 + placement: + constraints: + - node.labels.hla.state == true + restart_policy: + condition: on-failure + resources: + limits: + cpus: "1.00" + memory: 768M + + redis: + image: redis:7.4-bookworm + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 30 + networks: [backend] + deploy: + replicas: 1 + placement: + constraints: + - node.labels.hla.state == true + restart_policy: + condition: on-failure + resources: + limits: + cpus: "0.25" + memory: 256M + +networks: + backend: + driver: overlay + attachable: true + +volumes: + postgres-data: + redis-data: diff --git a/tests/resilience.py b/tests/resilience.py index f3fa700a..016c18b0 100644 --- a/tests/resilience.py +++ b/tests/resilience.py @@ -100,7 +100,8 @@ def main(): check(first_recovery is not None and first_recovery<=10,'writes did not recover within 10 seconds from hook start') check(len(after_instances)>=5,'too few successful writes after stop') check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped') - print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False)) + duration = time.monotonic()-started + print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'successful_rps':round(len(successes)/duration,3),'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False)) # Existing token, no re-login. Read all confirmations and reject duplicates. messages = history(c,bob,chat) ids = [m['id'] for m in messages] -- 2.54.0 From 7080fdd3e28326f4f756a70f504efc5c3a13a7bb Mon Sep 17 00:00:00 2001 From: EgorAnti Date: Tue, 3 Nov 2026 21:00:57 +0300 Subject: [PATCH 2/3] =?UTF-8?q?=D0=97=D0=B0=D0=BF=D0=BE=D0=BB=D0=BD=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D0=BE=D0=BF=D0=B8=D1=81=D0=B0=D0=BD=D0=B8=D0=B5?= =?UTF-8?q?=20pull=20request?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/pull_request_template.md | 91 +++++++++++++++++++++++++++----- 1 file changed, 77 insertions(+), 14 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e6e63eb2..59e33ff6 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,81 @@ ## Лабораторная 3 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Антипин Егор Иванович, БПИ2301 -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Целевая оценка (3/4/5): 5 -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Трек 5, если заявлен: трек 2 — «Отказ узла» + +Что реализовано и что перенесено: + +- Из лабораторной №2 перенесены Spring Boot API, PostgreSQL-схема пользователей, сессий, чатов и сообщений, Bearer-аутентификация, Redis cache-aside и Flyway-миграции. +- Основной стенд описан в `compose.yaml`: Nginx является единственной публичной точкой входа, за ним работают две stateless API-реплики с общей PostgreSQL и общим Redis. +- Сессии и все данные, влияющие на корректность, хранятся в PostgreSQL; sticky session не используется. Реплики наблюдаются по `X-Instance-Id`. +- Миграции вынесены в отдельный one-shot service `migrate`; в API-репликах Flyway отключён. +- Настроены `least_conn`, динамическое обновление Docker DNS, readiness, таймауты, graceful shutdown и пассивное исключение недоступного backend. +- Реализована общая идемпотентность отправки сообщения через `message_idempotency`, SHA-256 тела и транзакционную PostgreSQL advisory-блокировку. +- Добавлены action scripts остановки/восстановления выбранной API-реплики, последовательный failover обеих реплик и rolling replacement под нагрузкой. +- Для трека 5 добавлен отдельный трёхузловой Swarm-стенд: один manager, два workers, локальный registry, перенос API task при жёсткой потере worker и проверка сохранности подтверждённых сообщений. +- Дополнительные проверки находятся в `student_tests`; исходные публичные тесты остаются в `tests`, Spring-тесты — в `src/test/java`. + +Команды запуска из чистого clone: + +```sh +cp .env.example .env +# задать локальный DATABASE_PASSWORD в .env +docker compose --env-file .env config --quiet +docker compose --env-file .env build api +docker compose --env-file .env up -d postgres redis +docker compose --env-file .env run --rm migrate +docker compose --env-file .env up -d --scale api=2 api nginx +curl --fail http://localhost:8080/health/ready +``` + +Проверки основного стенда: + +```sh +make check +make test +make test-idempotency +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover-each ACTION_SCRIPT=scripts/stop-one.sh RESTORE_SCRIPT=scripts/restore-api.sh +make test-rolling ACTION_SCRIPT=scripts/rolling-update.sh +``` + +Трёхузловой Swarm-эксперимент на Windows: + +```powershell +make swarm-up +make test-node-failover PYTHON=python +make swarm-down +``` + +Результаты проверок и ссылка на REPORT.md: + +- `make check` — PASS; основной `make test` через Nginx — `PASS: lab3 public API scenario`. +- Конкурентная идемпотентность — 32 одновременных запроса через две реплики создали одно сообщение. +- Последовательная остановка обеих Compose-реплик — 0 ошибок, 10,839/10,014 successful RPS, первая успешная операция через 1,306/2,672 s. +- Rolling replacement — четыре instance ID, 12 147 успешных записей, 177 ошибок, 67,439 successful RPS, error ratio 1,4362%. +- Потеря Swarm worker — API task перенесена с `worker2` на `worker1`; 165 чтений, 166 записей, 0 ошибок, первая успешная операция через 1,796 s; подтверждённые данные сохранены. +- Полные команды, методика, raw JSON и объяснение архитектуры приведены в [REPORT.md](https://gitea.privcloud.tech/Egor_Antipin/mtusi2026lab3/src/branch/solution/bpi2301-antipin/REPORT.md) и каталоге `evidence/`. + +Ограничения / известные ошибки: + +- PostgreSQL не реплицирован и остаётся SPOF; отказ его узла не проверялся и не заявляется (`database_node_failed=false`). +- Nginx, единственный Swarm manager, Redis, локальные volumes, underlay и физический Docker host также остаются SPOF. +- Nginx OSS использует пассивное runtime-исключение backend, а не активный health-check upstream. +- Во время rolling replacement зафиксировано 1,4362% кратковременных ошибок; они намеренно не скрыты клиентским retry. +- Swarm-опыт выполнен на DinD-узлах одного Windows/Docker Desktop host и не моделирует потерю разных физических серверов. + +Источники и использование ИИ: + +- Использованы исходный skeleton лабораторной, её OpenAPI-контракт и стандартные open-source зависимости проекта: Spring Boot, PostgreSQL, Redis, Flyway, Nginx, Docker Compose и Docker Swarm. +- ИИ применялся для обсуждения архитектуры Compose/Nginx, readiness и graceful shutdown, проектирования общей идемпотентности, подготовки дополнительных тестов и Swarm-сценария, анализа измерений и оформления документации. +- Прикладной код готового стороннего сервера не заимствовался. Утверждения о поведении проверены запускаемыми сценариями и сохранёнными raw-результатами в `evidence/`. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. -- 2.54.0 From 9c3e4fd1260fab215c539d44a16ddd036341e7ee Mon Sep 17 00:00:00 2001 From: EgorAnti Date: Thu, 5 Nov 2026 12:48:30 +0300 Subject: [PATCH 3/3] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20readiness=20LB=20=D0=B8=20=D1=80=D0=B5=D1=81?= =?UTF-8?q?=D1=83=D1=80=D1=81=D0=BD=D1=8B=D0=B5=20=D0=BB=D0=B8=D0=BC=D0=B8?= =?UTF-8?q?=D1=82=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/pull_request_template.md | 7 +- Makefile | 5 +- README.md | 20 ++++ REPORT.md | 30 ++++- compose.yaml | 6 + evidence/readiness-and-limits-2026-11-05.md | 32 ++++++ infra/README.md | 16 ++- infra/nginx.conf | 14 +++ scripts/check_contract.py | 4 +- scripts/restart.example.sh | 0 scripts/restart.sh | 0 scripts/restore-api.sh | 0 scripts/rolling-update.sh | 0 scripts/stop-one.example.sh | 0 scripts/stop-one.sh | 0 scripts/swarm-lab-up.ps1 | 1 + .../config/SecurityConfig.java | 17 ++- .../controller/HealthController.java | 42 ++----- .../service/ReadinessService.java | 81 ++++++++++++++ .../web/ReadinessGuardFilter.java | 56 ++++++++++ .../web/ReadinessGuardFilterTest.java | 62 +++++++++++ student_tests/README.md | 3 + student_tests/readiness_gate.py | 104 ++++++++++++++++++ swarm/nginx.conf | 11 ++ 24 files changed, 466 insertions(+), 45 deletions(-) create mode 100644 evidence/readiness-and-limits-2026-11-05.md mode change 100644 => 100755 scripts/restart.example.sh mode change 100644 => 100755 scripts/restart.sh mode change 100644 => 100755 scripts/restore-api.sh mode change 100644 => 100755 scripts/rolling-update.sh mode change 100644 => 100755 scripts/stop-one.example.sh mode change 100644 => 100755 scripts/stop-one.sh create mode 100644 src/main/java/com/example/mtusi_messenger/service/ReadinessService.java create mode 100644 src/main/java/com/example/mtusi_messenger/web/ReadinessGuardFilter.java create mode 100644 src/test/java/com/example/mtusi_messenger/web/ReadinessGuardFilterTest.java create mode 100644 student_tests/readiness_gate.py diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 59e33ff6..77f2b940 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -12,7 +12,7 @@ - Основной стенд описан в `compose.yaml`: Nginx является единственной публичной точкой входа, за ним работают две stateless API-реплики с общей PostgreSQL и общим Redis. - Сессии и все данные, влияющие на корректность, хранятся в PostgreSQL; sticky session не используется. Реплики наблюдаются по `X-Instance-Id`. - Миграции вынесены в отдельный one-shot service `migrate`; в API-репликах Flyway отключён. -- Настроены `least_conn`, динамическое обновление Docker DNS, readiness, таймауты, graceful shutdown и пассивное исключение недоступного backend. +- Настроены `least_conn`, динамическое обновление Docker DNS, readiness guard, таймауты, graceful shutdown и пассивное исключение backend по 503. - Реализована общая идемпотентность отправки сообщения через `message_idempotency`, SHA-256 тела и транзакционную PostgreSQL advisory-блокировку. - Добавлены action scripts остановки/восстановления выбранной API-реплики, последовательный failover обеих реплик и rolling replacement под нагрузкой. - Для трека 5 добавлен отдельный трёхузловой Swarm-стенд: один manager, два workers, локальный registry, перенос API task при жёсткой потере worker и проверка сохранности подтверждённых сообщений. @@ -37,8 +37,10 @@ curl --fail http://localhost:8080/health/ready make check make test make test-idempotency +make test-readiness-gate make test-persistence ACTION_SCRIPT=scripts/restart.sh make test-failover-each ACTION_SCRIPT=scripts/stop-one.sh RESTORE_SCRIPT=scripts/restore-api.sh +chmod +x scripts/rolling-update.sh make test-rolling ACTION_SCRIPT=scripts/rolling-update.sh ``` @@ -54,6 +56,7 @@ make swarm-down - `make check` — PASS; основной `make test` через Nginx — `PASS: lab3 public API scenario`. - Конкурентная идемпотентность — 32 одновременных запроса через две реплики создали одно сообщение. +- Readiness gate — после временного переименования V3-таблицы обе API стали `unhealthy`, readiness и `/users/me` через LB вернули JSON 503; после восстановления вернулись `healthy`/200. - Последовательная остановка обеих Compose-реплик — 0 ошибок, 10,839/10,014 successful RPS, первая успешная операция через 1,306/2,672 s. - Rolling replacement — четыре instance ID, 12 147 успешных записей, 177 ошибок, 67,439 successful RPS, error ratio 1,4362%. - Потеря Swarm worker — API task перенесена с `worker2` на `worker1`; 165 чтений, 166 записей, 0 ошибок, первая успешная операция через 1,796 s; подтверждённые данные сохранены. @@ -63,7 +66,7 @@ make swarm-down - PostgreSQL не реплицирован и остаётся SPOF; отказ его узла не проверялся и не заявляется (`database_node_failed=false`). - Nginx, единственный Swarm manager, Redis, локальные volumes, underlay и физический Docker host также остаются SPOF. -- Nginx OSS использует пассивное runtime-исключение backend, а не активный health-check upstream. +- Nginx OSS использует пассивное исключение, поэтому API readiness guard возвращает 503 на бизнес-запрос неготовой реплики; Nginx повторяет его на другом peer, а при отсутствии готовых API возвращается 503. - Во время rolling replacement зафиксировано 1,4362% кратковременных ошибок; они намеренно не скрыты клиентским retry. - Swarm-опыт выполнен на DinD-узлах одного Windows/Docker Desktop host и не моделирует потерю разных физических серверов. diff --git a/Makefile b/Makefile index c1d00b88..2286fe9e 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test test-idempotency load test-persistence test-failover test-failover-each test-rolling swarm-up test-node-failover swarm-down +.PHONY: help check validate test test-idempotency test-readiness-gate load test-persistence test-failover test-failover-each test-rolling swarm-up test-node-failover swarm-down help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -21,6 +21,9 @@ test: test-idempotency: $(PYTHON) student_tests/idempotency_concurrent.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-readiness-gate: + $(PYTHON) student_tests/readiness_gate.py --base-url "$(BASE_URL)" $(TLS_ARGS) + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/README.md b/README.md index 1ec655ca..e3f2c5f3 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,25 @@ hostname контейнера и останавливает только эту docker compose --env-file .env up -d --scale api=2 api ``` +Связь readiness с обслуживанием бизнес-запросов проверяется отдельно. Docker +вызывает `/health/ready` каждые 2 секунды; неуспешная проверка общей схемы +переводит конкретный Spring-процесс в `REFUSING_TRAFFIC`. Встроенный фильтр +возвращает 503 для всех бизнес-endpoint этой реплики до успешного probe. Nginx +повторяет такой ответ на другом peer и пассивно исключает отказавший адрес. Если +готовых API нет, `error_page` нормализует финальный ответ LB в JSON 503 с +`X-Request-Id`/`X-Instance-Id`, а не в HTML 502 или устаревший успешный ответ. +Первый dependency failure обнаруживается не позднее чем за 7 секунд: до 2 s до +следующего probe плюс SQL query timeout 5 s. Docker помечает контейнер +`unhealthy` после 15 последовательных отказов, то есть примерно через 30 s, +но readiness guard прекращает бизнес-трафик уже после первой ошибки probe. + +Воспроизведение с временным переименованием V3-таблицы и гарантированным +восстановлением в `finally`: + +```sh +make test-readiness-gate +``` + ### Проверки уровня 4 Обе реплики последовательно останавливаются с восстановлением между опытами: @@ -81,6 +100,7 @@ successful RPS, число ошибок и время первой успешн следующей. Проверка одновременно держит четыре потока записи: ```sh +chmod +x scripts/rolling-update.sh make test-rolling ACTION_SCRIPT=scripts/rolling-update.sh ``` diff --git a/REPORT.md b/REPORT.md index d1f696a4..055486f6 100644 --- a/REPORT.md +++ b/REPORT.md @@ -57,8 +57,8 @@ Nginx — единственный сервис, публикующий порт | Публичный LB | `nginx`, `nginx:1.27.4-bookworm` | `nginx -t`, каждые 2 s | нет | `127.0.0.1:8080 -> 80`, `backend` | не требуются | | API | `api`, локальный `hla-lab3-api:local` | `GET /health/ready`, каждые 2 s | нет | только `8080/backend` | параметры БД из `.env` через environment | | Миграции | `migrate`, тот же образ API | one-shot, код завершения | нет | `backend` | те же параметры БД | -| Источник истины | `postgres`, `postgres:16.4-bookworm` | `pg_isready`, каждые 2 s | `postgres-data` | только `5432/backend` | `DATABASE_PASSWORD` из `.env` | -| Общий кэш | `redis`, `redis:7.4-bookworm` | `redis-cli ping`, каждые 2 s | `redis-data`, AOF everysec | только `6379/backend` | наружу порт не открыт | +| Источник истины | `postgres`, `postgres:16.4-bookworm` | `pg_isready`, каждые 2 s | `postgres-data` | только `5432/backend`; 1 CPU/768 MiB | `DATABASE_PASSWORD` из `.env` | +| Общий кэш | `redis`, `redis:7.4-bookworm` | `redis-cli ping`, каждые 2 s | `redis-data`, AOF everysec | только `6379/backend`; 0,25 CPU/256 MiB | наружу порт не открыт | PostgreSQL хранит пользователей, Bearer-сессии, чаты, членство, сообщения и `message_idempotency`. Поэтому следующий запрос одного пользователя может @@ -133,9 +133,16 @@ Sticky session не используется. Она закрепляла бы Ошибка соединения определяется быстрее — в пределах 500 ms на попытку. Это укладывается в требование возобновить новые операции не позднее 10 секунд. -Nginx OSS использует пассивную runtime-проверку upstream: Docker readiness не -допускает ранний операторский запуск публичного маршрута, а отказ уже -работающего backend обнаруживается запросом и указанными таймерами. +Nginx OSS использует пассивную runtime-проверку upstream. Чтобы Docker health +не был отдельным от реального обслуживания, неуспешный schema probe переводит +Spring-процесс в `REFUSING_TRAFFIC`. Readiness guard этой реплики отвечает 503 +на любой бизнес-запрос до успешного восстановления. Nginx повторяет 503 на +другом peer и учитывает его как failure; если обе реплики неготовы, успешный +бизнес-ответ невозможен. Named location `@no_ready_api` преобразует финальный +Nginx 502/503/504 в контрактный JSON 503 с обязательными tracing headers. +Dependency failure обнаруживается максимум за 7 s: до 2 s до следующего probe +и до 5 s SQL query timeout. Docker ставит `unhealthy` после 15 ошибок, примерно +через 30 s, но guard блокирует бизнес-трафик после первой ошибки probe. ## 6. Readiness и graceful shutdown @@ -157,6 +164,14 @@ Docker вызывает readiness каждые 2 s, timeout — 2 s, start perio `stop_grace_period`. У Nginx `worker_shutdown_timeout=10s` и такой же `stop_grace_period`. +Регрессия исходного дефекта проверяется `make test-readiness-gate`: тест +временно переименовывает `message_idempotency`, ждёт `unhealthy` обеих API, +проверяет 503 от readiness и авторизованного `/users/me`, восстанавливает +таблицу в `finally` и требует возврата обеих реплик в `healthy`. +Фактический запуск 05.11.2026: обе реплики стали `unhealthy`, readiness и +авторизованный `/api/v1/users/me` через LB вернули JSON 503; после восстановления +таблицы обе реплики стали `healthy`, а те же запросы вернули 200. + ## 7. Общая идемпотентность сообщений Для `POST /api/v1/chats/{chatId}/messages` обязателен `Idempotency-Key`. @@ -237,6 +252,9 @@ PostgreSQL, а не память API и не Redis. - каждая Compose API-реплика: 0,75 CPU и 768 MiB; - Nginx: 0,25 CPU и 128 MiB; +- PostgreSQL: 1 CPU и 768 MiB; +- Redis: 0,25 CPU и 256 MiB; +- one-shot `migrate`: 0,75 CPU и 768 MiB; - API не публикует host port; - volumes есть только у PostgreSQL и Redis; - `container_name` отсутствует; @@ -251,6 +269,7 @@ PostgreSQL, а не память API и не Redis. четыре закрытых потока записи: ```sh +chmod +x scripts/rolling-update.sh make test-rolling ACTION_SCRIPT=scripts/rolling-update.sh ``` @@ -368,6 +387,7 @@ storage или физического host не переживается и не | Структура и синтаксис | `make check` | PASS | | Основной API через LB | `make test` | PASS | | Общая идемпотентность | `make test-idempotency` | 32 запроса, 2 реплики, 1 сообщение | +| Readiness блокирует бизнес-трафик | `make test-readiness-gate` | обе unhealthy-реплики отвечают 503 и восстанавливаются | | Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | прежние token и данные доступны | | Выбранная реплика | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | чтение/запись продолжаются | | Каждая реплика | `make test-failover-each ...` | 0 ошибок, recovery 1,306/2,672 s | diff --git a/compose.yaml b/compose.yaml index de14a7f5..b7ecc88f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -37,6 +37,8 @@ services: timeout: 2s retries: 30 restart: unless-stopped + cpus: 1.0 + mem_limit: 768m networks: [backend] redis: @@ -50,6 +52,8 @@ services: timeout: 2s retries: 30 restart: unless-stopped + cpus: 0.25 + mem_limit: 256m networks: [backend] # One-shot Flyway runner. It is invoked explicitly before API replicas. @@ -66,6 +70,8 @@ services: redis: condition: service_healthy restart: "no" + cpus: 0.75 + mem_limit: 768m networks: [backend] api: diff --git a/evidence/readiness-and-limits-2026-11-05.md b/evidence/readiness-and-limits-2026-11-05.md new file mode 100644 index 00000000..03d6081c --- /dev/null +++ b/evidence/readiness-and-limits-2026-11-05.md @@ -0,0 +1,32 @@ +# Readiness gate and Compose limits — 2026-11-05 + +Command: + +```text +python -B student_tests/readiness_gate.py --base-url http://localhost:8080 +``` + +Observed result: + +```text +PASS: 2 unhealthy API containers refused business traffic with 503 and recovered after schema restoration +table_state=true|false +``` + +The test temporarily renamed `message_idempotency`, waited until both Docker +API health states became `unhealthy`, and verified JSON 503 from both +`/health/ready` and authenticated `/api/v1/users/me` through Nginx. Its +`finally` block restored the table. Both API containers then became `healthy`, +and readiness/business requests returned 200. + +Applied `docker inspect` values: + +| Service | NanoCpus | Memory bytes | +| --- | ---: | ---: | +| PostgreSQL | 1,000,000,000 | 805,306,368 | +| Redis | 250,000,000 | 268,435,456 | +| API (each) | 750,000,000 | 805,306,368 | +| Nginx | 250,000,000 | 134,217,728 | + +The one-shot Compose migration service is configured with 0.75 CPU and 768 MiB. +The Swarm migration job uses the same explicit limits. diff --git a/infra/README.md b/infra/README.md index 262afec2..25c63559 100644 --- a/infra/README.md +++ b/infra/README.md @@ -21,6 +21,18 @@ connect timeout 500 ms, read/send timeout 4 s, две upstream-попытки и Docker DNS с TTL 2 s. Максимальное обнаружение зависшего backend — 4 s; максимум двух последовательных timeout-попыток — около 8.5 s. +Неуспешный schema probe публикует Spring-состояние `REFUSING_TRAFFIC`, и +readiness guard до аутентификации отвечает 503 на любой бизнес-запрос этой +реплики. Nginx настроен повторять 503 на другом peer и учитывать ответ как +пассивный failure. Поэтому Docker health не используется как недоступный Nginx +API: готовность связана с фактическим ответом backend. При восстановлении схемы +следующий probe возвращает процесс в `ACCEPTING_TRAFFIC`. +Когда готовы ноль peer, named location `@no_ready_api` преобразует внутренний +Nginx 502/финальный upstream 503/504 в контрактный JSON 503. +Граница обнаружения dependency failure — 7 s (период 2 s плюс query timeout +5 s). Docker status становится `unhealthy` примерно через 30 s из-за +`retries=15`, но это не задерживает readiness guard. + Flyway разрешён только в one-shot service `migrate`. У API задано `FLYWAY_ENABLED=false`, поэтому запуск нескольких реплик не создаёт гонку инициализации схемы. @@ -29,7 +41,9 @@ Restart hook поднимает зависимости по фазам и не API-реплики не стали healthy. Это не позволяет Docker DNS объявить прогревающуюся реплику через уже доступную публичную точку входа. -API ограничен 0.75 CPU и 768 MiB на контейнер, Nginx — 0.25 CPU и 128 MiB. +PostgreSQL ограничен 1 CPU и 768 MiB, Redis — 0.25 CPU и 256 MiB, one-shot +migrate — 0.75 CPU и 768 MiB, API — 0.75 CPU и 768 MiB на контейнер, Nginx — +0.25 CPU и 128 MiB. Порт публикует только Nginx. PostgreSQL и Redis имеют собственные volumes; API и Nginx постоянных volumes не имеют. `container_name` не используется, поэтому один Compose-service `api` масштабируется параметром `--scale`. diff --git a/infra/nginx.conf b/infra/nginx.conf index 04ed5162..16050ab7 100644 --- a/infra/nginx.conf +++ b/infra/nginx.conf @@ -24,6 +24,8 @@ http { server { listen 80; server_name _; + proxy_intercept_errors on; + error_page 502 503 504 = @no_ready_api; location = /nginx-health { access_log off; @@ -31,6 +33,18 @@ http { return 200 '{"status":"ok"}'; } + # After both peers have refused traffic or become unreachable, Nginx + # would normally emit its own HTML 502. Keep the public API contract: + # no ready backend is a JSON 503 with the required tracing headers. + location @no_ready_api { + internal; + default_type application/json; + add_header X-Request-Id $request_id always; + add_header X-Instance-Id nginx always; + add_header Cache-Control "no-store" always; + return 503 '{"error":{"code":"unavailable","message":"No ready API backend","request_id":"$request_id"}}'; + } + # Message POST is the only non-idempotent HTTP method that is safe to # replay here: the API requires Idempotency-Key and commits the message # together with its shared PostgreSQL idempotency record. diff --git a/scripts/check_contract.py b/scripts/check_contract.py index dcb34dfa..adb4b9e1 100644 --- a/scripts/check_contract.py +++ b/scripts/check_contract.py @@ -40,7 +40,9 @@ def main(): assert 'X-Instance-Id' in response['headers'] assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2) assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4) - for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')): + for file in (list((ROOT/'tests').glob('*.py')) + + list((ROOT/'student_tests').glob('*.py')) + + list((ROOT/'scripts').glob('*.py'))): ast.parse(file.read_text(),filename=str(file)) for file in ROOT.rglob('*.json'): if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts): diff --git a/scripts/restart.example.sh b/scripts/restart.example.sh old mode 100644 new mode 100755 diff --git a/scripts/restart.sh b/scripts/restart.sh old mode 100644 new mode 100755 diff --git a/scripts/restore-api.sh b/scripts/restore-api.sh old mode 100644 new mode 100755 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh old mode 100644 new mode 100755 diff --git a/scripts/stop-one.example.sh b/scripts/stop-one.example.sh old mode 100644 new mode 100755 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh old mode 100644 new mode 100755 diff --git a/scripts/swarm-lab-up.ps1 b/scripts/swarm-lab-up.ps1 index 412af524..c010ed19 100644 --- a/scripts/swarm-lab-up.ps1 +++ b/scripts/swarm-lab-up.ps1 @@ -112,6 +112,7 @@ Invoke-Checked docker exec $manager docker service create --name hla-migrate ` --mode replicated-job --replicas 1 --max-concurrent 1 ` --network hla-state_backend --constraint node.labels.hla.state==true ` --restart-condition on-failure --restart-delay 3s ` + --limit-cpu 0.75 --limit-memory 768M ` --env "DATABASE_URL=jdbc:postgresql://hla-state_postgres:5432/$database" ` --env "DATABASE_USER=$user" --env "DATABASE_PASSWORD=$password" ` --env FLYWAY_ENABLED=true --env MIGRATE_ONLY=true ` diff --git a/src/main/java/com/example/mtusi_messenger/config/SecurityConfig.java b/src/main/java/com/example/mtusi_messenger/config/SecurityConfig.java index e38246af..f2b9ae41 100644 --- a/src/main/java/com/example/mtusi_messenger/config/SecurityConfig.java +++ b/src/main/java/com/example/mtusi_messenger/config/SecurityConfig.java @@ -4,6 +4,7 @@ import com.example.mtusi_messenger.security.ApiAuthenticationFilter; import com.example.mtusi_messenger.security.DatabaseAuthenticationProvider; import com.example.mtusi_messenger.security.RestAuthenticationEntryPoint; import com.example.mtusi_messenger.service.AuthenticationService; +import com.example.mtusi_messenger.web.ReadinessGuardFilter; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication; @@ -61,10 +62,20 @@ public class SecurityConfig { return registration; } + @Bean + FilterRegistrationBean disableReadinessGuardServletRegistration( + ReadinessGuardFilter filter + ) { + var registration = new FilterRegistrationBean<>(filter); + registration.setEnabled(false); + return registration; + } + @Bean SecurityFilterChain securityFilterChain( HttpSecurity http, ApiAuthenticationFilter apiAuthenticationFilter, + ReadinessGuardFilter readinessGuardFilter, @Qualifier("bearerAuthenticationEntryPoint") RestAuthenticationEntryPoint bearerAuthenticationEntryPoint ) throws Exception { http @@ -86,7 +97,11 @@ public class SecurityConfig { ).permitAll() .requestMatchers(HttpMethod.POST, "/api/v1/users").permitAll() .anyRequest().authenticated()) - .addFilterBefore(apiAuthenticationFilter, AnonymousAuthenticationFilter.class); + // Refuse business traffic before authentication touches storage. + // Nginx retries this 503 on another peer and passively excludes + // the refusing replica according to max_fails/fail_timeout. + .addFilterBefore(readinessGuardFilter, AnonymousAuthenticationFilter.class) + .addFilterAfter(apiAuthenticationFilter, ReadinessGuardFilter.class); return http.build(); } } diff --git a/src/main/java/com/example/mtusi_messenger/controller/HealthController.java b/src/main/java/com/example/mtusi_messenger/controller/HealthController.java index 2b9abc35..e4bfec30 100644 --- a/src/main/java/com/example/mtusi_messenger/controller/HealthController.java +++ b/src/main/java/com/example/mtusi_messenger/controller/HealthController.java @@ -1,31 +1,18 @@ package com.example.mtusi_messenger.controller; import org.springframework.http.ResponseEntity; -import org.springframework.dao.DataAccessException; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.data.redis.core.StringRedisTemplate; -import org.springframework.data.redis.core.RedisCallback; -import org.springframework.boot.availability.ApplicationAvailability; -import org.springframework.boot.availability.ReadinessState; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; import com.example.mtusi_messenger.dto.HealthResponse; import com.example.mtusi_messenger.exception.ApiExceptions; +import com.example.mtusi_messenger.service.ReadinessService; @RestController public class HealthController { - private final JdbcTemplate jdbc; - private final StringRedisTemplate redis; - private final ApplicationAvailability availability; + private final ReadinessService readiness; - public HealthController( - JdbcTemplate jdbc, - StringRedisTemplate redis, - ApplicationAvailability availability - ) { - this.jdbc = jdbc; - this.redis = redis; - this.availability = availability; + public HealthController(ReadinessService readiness) { + this.readiness = readiness; } @GetMapping("/health/live") @@ -35,24 +22,11 @@ public class HealthController { @GetMapping("/health/ready") public ResponseEntity ready() { - if (availability.getReadinessState() != ReadinessState.ACCEPTING_TRAFFIC) { + // Always probe dependencies, even after a failure, so a restored schema + // can move this process back to ACCEPTING_TRAFFIC without a restart. + if (!readiness.probeDependencies() || !readiness.acceptsBusinessTraffic()) { throw ApiExceptions.unavailable("Application is not accepting traffic"); } - try { - // Warm the lazy Lettuce connection on every replica. Redis remains - // an optional cache: failure is ignored and business reads fall - // back to PostgreSQL. - redis.execute((RedisCallback) connection -> connection.ping()); - } catch (DataAccessException ignored) { - // Optional cache unavailable; PostgreSQL remains authoritative. - } - try { - // A schema-level probe prevents a replica from becoming ready - // before the separately executed migration job has completed. - jdbc.queryForObject("SELECT count(*) FROM message_idempotency", Long.class); - return ResponseEntity.ok(new HealthResponse("ok")); - } catch (DataAccessException e) { - throw ApiExceptions.unavailable("Database is unavailable"); - } + return ResponseEntity.ok(new HealthResponse("ok")); } } diff --git a/src/main/java/com/example/mtusi_messenger/service/ReadinessService.java b/src/main/java/com/example/mtusi_messenger/service/ReadinessService.java new file mode 100644 index 00000000..7600c4d9 --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/ReadinessService.java @@ -0,0 +1,81 @@ +package com.example.mtusi_messenger.service; + +import org.springframework.boot.availability.ApplicationAvailability; +import org.springframework.boot.availability.AvailabilityChangeEvent; +import org.springframework.boot.availability.ReadinessState; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.dao.DataAccessException; +import org.springframework.data.redis.core.RedisCallback; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; + +import java.util.concurrent.atomic.AtomicBoolean; + +@Service +public class ReadinessService { + private final JdbcTemplate jdbc; + private final StringRedisTemplate redis; + private final ApplicationAvailability availability; + private final ApplicationEventPublisher events; + private final AtomicBoolean dependencyUnavailable = new AtomicBoolean(false); + private final AtomicBoolean shuttingDown = new AtomicBoolean(false); + + public ReadinessService( + JdbcTemplate jdbc, + StringRedisTemplate redis, + ApplicationAvailability availability, + ApplicationEventPublisher events + ) { + this.jdbc = jdbc; + this.redis = redis; + this.availability = availability; + this.events = events; + } + + public boolean probeDependencies() { + try { + // Redis is an optional cache. Ping only warms the lazy connection; + // its failure must not make the authoritative API unavailable. + redis.execute((RedisCallback) connection -> connection.ping()); + } catch (DataAccessException ignored) { + // PostgreSQL remains the source of truth. + } + + try { + // The schema-level probe also proves that the separate V3 migration + // completed before this replica is allowed to serve business traffic. + jdbc.queryForObject("SELECT count(*) FROM message_idempotency", Long.class); + markDependenciesAvailable(); + return true; + } catch (DataAccessException e) { + markDependenciesUnavailable(); + return false; + } + } + + public boolean acceptsBusinessTraffic() { + return !shuttingDown.get() + && !dependencyUnavailable.get() + && availability.getReadinessState() == ReadinessState.ACCEPTING_TRAFFIC; + } + + @EventListener(ContextClosedEvent.class) + void onContextClosed() { + shuttingDown.set(true); + } + + private void markDependenciesUnavailable() { + dependencyUnavailable.set(true); + AvailabilityChangeEvent.publish(events, this, ReadinessState.REFUSING_TRAFFIC); + } + + private void markDependenciesAvailable() { + boolean recovered = dependencyUnavailable.getAndSet(false); + if (recovered && !shuttingDown.get()) { + AvailabilityChangeEvent.publish(events, this, ReadinessState.ACCEPTING_TRAFFIC); + } + } +} diff --git a/src/main/java/com/example/mtusi_messenger/web/ReadinessGuardFilter.java b/src/main/java/com/example/mtusi_messenger/web/ReadinessGuardFilter.java new file mode 100644 index 00000000..c69594ea --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/web/ReadinessGuardFilter.java @@ -0,0 +1,56 @@ +package com.example.mtusi_messenger.web; + +import com.example.mtusi_messenger.dto.ErrorResponse; +import com.example.mtusi_messenger.service.ReadinessService; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; + +@Component +public class ReadinessGuardFilter extends OncePerRequestFilter { + private final ReadinessService readiness; + private final ObjectMapper mapper; + + public ReadinessGuardFilter(ReadinessService readiness, ObjectMapper mapper) { + this.readiness = readiness; + this.mapper = mapper; + } + + @Override + protected boolean shouldNotFilter(HttpServletRequest request) { + String path = request.getRequestURI(); + return path.startsWith("/health/") + || path.equals("/swagger-ui.html") + || path.startsWith("/swagger-ui/") + || path.startsWith("/v3/api-docs/") + || path.equals("/v3/api-docs"); + } + + @Override + protected void doFilterInternal( + HttpServletRequest request, + HttpServletResponse response, + FilterChain filterChain + ) throws ServletException, IOException { + if (readiness.acceptsBusinessTraffic()) { + filterChain.doFilter(request, response); + return; + } + + response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + String requestId = (String) request.getAttribute(RequestIdFilter.ATTRIBUTE); + mapper.writeValue(response.getOutputStream(), new ErrorResponse( + new ErrorResponse.ErrorBody( + "unavailable", + "Application is not ready to serve business traffic", + requestId))); + } +} diff --git a/src/test/java/com/example/mtusi_messenger/web/ReadinessGuardFilterTest.java b/src/test/java/com/example/mtusi_messenger/web/ReadinessGuardFilterTest.java new file mode 100644 index 00000000..a8374d13 --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/web/ReadinessGuardFilterTest.java @@ -0,0 +1,62 @@ +package com.example.mtusi_messenger.web; + +import com.example.mtusi_messenger.service.ReadinessService; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockFilterChain; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class ReadinessGuardFilterTest { + private final ReadinessService readiness = mock(ReadinessService.class); + private final ObjectMapper mapper = new ObjectMapper(); + private final ReadinessGuardFilter filter = new ReadinessGuardFilter(readiness, mapper); + + @Test + void refusingReplicaRejectsBusinessTrafficBeforeItReachesTheChain() throws Exception { + when(readiness.acceptsBusinessTraffic()).thenReturn(false); + var request = new MockHttpServletRequest("GET", "/api/v1/users/me"); + request.setAttribute(RequestIdFilter.ATTRIBUTE, "request-1"); + var response = new MockHttpServletResponse(); + var chain = new MockFilterChain(); + + filter.doFilter(request, response, chain); + + assertEquals(503, response.getStatus()); + assertNull(chain.getRequest()); + JsonNode body = mapper.readTree(response.getContentAsByteArray()); + assertEquals("unavailable", body.at("/error/code").asText()); + assertEquals("request-1", body.at("/error/request_id").asText()); + } + + @Test + void readyReplicaPassesBusinessTrafficToTheChain() throws Exception { + when(readiness.acceptsBusinessTraffic()).thenReturn(true); + var request = new MockHttpServletRequest("GET", "/api/v1/users/me"); + var response = new MockHttpServletResponse(); + var chain = new MockFilterChain(); + + filter.doFilter(request, response, chain); + + assertSame(request, chain.getRequest()); + } + + @Test + void healthProbeIsNeverBlockedByTheBusinessGuard() throws Exception { + when(readiness.acceptsBusinessTraffic()).thenReturn(false); + var request = new MockHttpServletRequest("GET", "/health/ready"); + var response = new MockHttpServletResponse(); + var chain = new MockFilterChain(); + + filter.doFilter(request, response, chain); + + assertSame(request, chain.getRequest()); + } +} diff --git a/student_tests/README.md b/student_tests/README.md index 0ac05425..a4b188ec 100644 --- a/student_tests/README.md +++ b/student_tests/README.md @@ -5,6 +5,9 @@ The course-provided black-box suite remains in `tests/`; Spring tests remain in `src/test/java`. - `idempotency_concurrent.py` — one key through multiple API replicas. +- `readiness_gate.py` — both replicas become unhealthy after the required + schema table disappears; readiness and business traffic must return 503, + then recover after the table is restored. - `failover_each.py` — sequential process failover of both Compose replicas. - `rolling_update.py` — sequential replacement under write load. - `node_failover.py` — hard loss of one worker in the disposable three-node diff --git a/student_tests/readiness_gate.py b/student_tests/readiness_gate.py new file mode 100644 index 00000000..1a9df4c6 --- /dev/null +++ b/student_tests/readiness_gate.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Reproduce and guard against serving business traffic from unready APIs.""" +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + +ORIGINAL_TABLE = 'message_idempotency' +TEMPORARY_TABLE = 'message_idempotency_readiness_test' + + +def command(*args): + result = subprocess.run( + args, + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False) + if result.returncode != 0: + output = result.stdout.strip() + raise Failure('command failed: ' + ' '.join(args[:3]) + + (': ' + output if output else '')) + return result.stdout.strip() + + +def rename_table(source, target): + sql = 'ALTER TABLE ' + source + ' RENAME TO ' + target + ';' + command( + 'docker', 'compose', 'exec', '-T', 'postgres', 'sh', '-ec', + 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" ' + '-d "$POSTGRES_DB" -c \'' + sql + '\'') + + +def api_container_ids(): + output = command('docker', 'compose', 'ps', '-q', 'api') + return [line.strip() for line in output.splitlines() if line.strip()] + + +def health_status(container_id): + return command( + 'docker', 'inspect', + '--format={{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}', + container_id) + + +def wait_for_all_health(expected, timeout): + deadline = time.monotonic() + timeout + last = [] + while time.monotonic() < deadline: + containers = api_container_ids() + last = [health_status(container) for container in containers] + if len(containers) >= 2 and all(value == expected for value in last): + return containers + time.sleep(1) + raise Failure('API health did not become ' + expected + + '; last statuses: ' + repr(last)) + + +def configure(parser): + parser.add_argument( + '--health-timeout', type=float, default=45.0, + help='seconds to wait for both Docker health states') + + +def main(): + args = arguments('Readiness must gate business traffic through the LB', configure) + client = Client(args.base_url, args.ca_file, timeout=8) + client.call('GET', '/health/ready', 200, schema='Health') + actor = client.user('readiness_gate') + + renamed = False + try: + rename_table(ORIGINAL_TABLE, TEMPORARY_TABLE) + renamed = True + containers = wait_for_all_health('unhealthy', args.health_timeout) + + client.call('GET', '/health/ready', 503) + instances = set() + for _ in range(8): + response = client.call( + 'GET', '/api/v1/users/me', 503, actor=actor) + instances.add(response['headers']['X-Instance-Id']) + check(instances, 'no refusing API instance was observed') + finally: + if renamed: + rename_table(TEMPORARY_TABLE, ORIGINAL_TABLE) + + wait_for_all_health('healthy', args.health_timeout) + # Nginx may retain a passive failure for fail_timeout=2s. + time.sleep(3) + client.call('GET', '/health/ready', 200, schema='Health') + client.call('GET', '/api/v1/users/me', 200, actor=actor, schema='User') + print('PASS: ' + str(len(containers)) + + ' unhealthy API containers refused business traffic with 503 ' + + 'and recovered after schema restoration') + + +if __name__ == '__main__': + run(main) diff --git a/swarm/nginx.conf b/swarm/nginx.conf index edd794c9..044151ce 100644 --- a/swarm/nginx.conf +++ b/swarm/nginx.conf @@ -21,12 +21,23 @@ http { server { listen 80; server_name _; + proxy_intercept_errors on; + error_page 502 503 504 = @no_ready_api; location = /nginx-health { default_type application/json; return 200 '{"status":"ok"}'; } + location @no_ready_api { + internal; + default_type application/json; + add_header X-Request-Id $request_id always; + add_header X-Instance-Id nginx always; + add_header Cache-Control "no-store" always; + return 503 '{"error":{"code":"unavailable","message":"No ready API backend","request_id":"$request_id"}}'; + } + location ~ ^/api/v1/chats/[0-9a-fA-F-]+/messages$ { proxy_pass http://api_backend; proxy_http_version 1.1; -- 2.54.0