From ee6e40cc4ecb34bfbcfa00ec5985bf35ea9e298c Mon Sep 17 00:00:00 2001 From: Mraksimus Date: Sat, 10 Oct 2026 00:47:27 +0300 Subject: [PATCH] Keep logout revocations until written to Redis --- REPORT.md | 6 +- evidence/gradle-tests.txt | 30 +++--- evidence/lost-tombstone.txt | 6 ++ .../dto/authentication/SessionRevocation.kt | 8 ++ .../authentication/SessionRevocationModel.kt | 11 +++ .../cache/RedisSessionCacheRepository.kt | 7 +- .../session/cache/SessionCacheRepository.kt | 5 +- .../ExposedSessionRevocationRepository.kt | 52 ++++++++++ .../revocation/SessionRevocationRepository.kt | 31 ++++++ .../DefaultAuthenticationService.kt | 95 ++++++++++++++++++- .../mtusi/messenger/main/plugins/Database.kt | 4 +- .../migrations/V2__session_revocations.sql | 6 ++ .../cache/RedisSessionCacheRepositoryTest.kt | 4 +- .../SessionLookupSingleFlightTest.kt | 17 +++- .../main/plugins/DependencyFailureTest.kt | 85 +++++++++++++++++ 15 files changed, 341 insertions(+), 26 deletions(-) create mode 100644 evidence/lost-tombstone.txt create mode 100644 src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/authentication/SessionRevocation.kt create mode 100644 src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/authentication/SessionRevocationModel.kt create mode 100644 src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/revocation/ExposedSessionRevocationRepository.kt create mode 100644 src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/revocation/SessionRevocationRepository.kt create mode 100644 src/main/resources/migrations/V2__session_revocations.sql diff --git a/REPORT.md b/REPORT.md index b711f71d..9bd74764 100644 --- a/REPORT.md +++ b/REPORT.md @@ -74,6 +74,7 @@ docker compose down -v - Чтение по схеме cache-aside: сначала Redis, если ключа нет — PostgreSQL, потом запись в Redis через `SET ... NX PXAT`. - Срок жизни ключа — меньшее из `expires_at` сессии и 60 секунд (`sessionCacheTtlSeconds`). Срок сессии к тому же сверяется в коде при каждом запросе, так что просроченная сессия не пройдёт, даже если ключ ещё жив. - Logout сначала удаляет сессию в PostgreSQL, потом пишет в ключ метку `revoked` до `expires_at`. Заполнение кэша идёт только через `NX`, поэтому запрос, который прочитал сессию из базы до logout и пишет её в кэш после, метку не перетрёт. +- Метка может не дойти до Redis, если он в этот момент лежит или завис, а копия сессии, закэшированная до сбоя, в Redis при этом останется. Поэтому logout в той же транзакции, что и удаление сессии, пишет строку в `session_revocations` (миграция `V2`) и удаляет её только после того, как метка записана. После любой ошибки Redis и при старте API кэшу не доверяют: чтения идут в базу, пока один запрос не допишет в Redis все метки из `session_revocations`. Неудачная попытка повторяется не чаще раза в секунду. - Если Redis недоступен (отказ соединения или таймаут 500 мс), чтение идёт мимо кэша в PostgreSQL, logout тоже работает: он нужен только базе. После неудачного подключения следующая попытка не раньше чем через секунду, а команды без соединения сразу отклоняются, поэтому лежащий Redis не тормозит запросы. - Промахи по одному токену схлопываются (single-flight): если поиск сессии в базе для этого токена уже идёт, следующие запросы ждут его результат, а не отправляют свой SELECT. Это защита от cache stampede, подробно в треке 1. Включается `SESSION_CACHE_SINGLE_FLIGHT` (по умолчанию `true`), выключатель оставлен для сравнения в эксперименте. - В лог пишется только смена состояния кэша («unavailable» и «available again»), а не каждый промах. Иначе при лежащем Redis API пишет по предупреждению на запрос. @@ -119,8 +120,9 @@ docker compose down -v | Rollback составной операции | `chat row is rolled back when inserting members fails` | см. строку про атомарность выше | evidence/gradle-tests.txt | | Индекс истории и EXPLAIN на ≥ 10 000 сообщений | `./scripts/explain-history.sh` | 70 000 сообщений, глубокая страница по курсору: Index Scan, 4 буфера, 0,03 мс. Без индекса: Seq Scan, 914 буферов. Разбор ниже | evidence/explain-history.txt | | Недоступность Redis | `docker compose stop redis`; `DependencyFailureTest` (отказ соединения и `docker pause`) | запросы с токеном 200, вход 201, logout 204 и токен сразу 401, анонимный запрос 401; при зависшем Redis запрос дольше максимум на один таймаут | evidence/failures.txt, evidence/gradle-tests.txt | +| Logout при недоступном Redis, потом рестарт API | `DependencyFailureTest`: `logout during a Redis outage is not undone by a stale cached copy`, `... survives an API restart`; то же на живом стенде (`docker compose pause redis`, logout, `unpause`, вернул в ключ старую копию сессии, `restart api`) | logout 204; после восстановления Redis и после рестарта API тот же токен 401 на чтение и на отправку сообщения, в ключе снова `revoked`, в логе `Wrote 1 pending session revocations to the cache`. Без исправления оба теста падали: токен получал 200 | evidence/lost-tombstone.txt, evidence/gradle-tests.txt | | Недоступность БД | `docker compose stop postgres`; `DependencyFailureTest` | `/health/live` 200, `/health/ready` 503, запросы с токеном и вход — 503 `unavailable` примерно за 2 с, без токена и с `X-User-Id` — 401. После старта базы тот же токен снова 200 | evidence/failures.txt | -| Свои автотесты | `./gradlew test` | 56 тестов, все проходят | evidence/gradle-tests.txt | +| Свои автотесты | `./gradlew test` | 58 тестов, все проходят | evidence/gradle-tests.txt | | Трек 1: cache stampede на конкурентном сценарии | `./scripts/track1/stampede.sh` с `VARIANT=off`, 200 VU, `FLUSHALL` раз в секунду | на каждую сессию за один сброс кэша в базу уходит 3,1 SELECT при 50 сессиях и 10,5 при 10 сессиях; запросов к `sessions` 4 651 и 3 138 за 30 с | evidence/track1/summary.md, evidence/track1/stampede/off/ | | Трек 1: защита и измерение до/после | то же с `VARIANT=on` (single-flight) | 0,97 SELECT на сессию за сброс в обоих случаях: запросов к `sessions` в 3,2 и 10,8 раза меньше. RPS и p50/p95/p99 в пределах шума | evidence/track1/summary.md, evidence/track1/stampede-runs.csv | | Трек 1: read-after-write из контракта | `./scripts/track1/read-after-write.sh`: 300 итераций, 16 параллельных читателей, logout посреди первого заполнения кэша; с защитой и без, Redis работает и остановлен | 1 200 итераций, 1,53 млн запросов, начатых после ответа 204 на logout: ни одного 200. Первый запрос после входа всегда 200. Через секунду ни один отозванный токен не ожил | evidence/track1/read-after-write/ | @@ -260,7 +262,7 @@ Redis у меня без persistence, поэтому после рестарта ## Ограничения и следующие шаги -- Если logout пришёлся на время, когда Redis лежал, метка `revoked` не записывается. Тогда копия сессии, закэшированная до сбоя, после возвращения Redis может прожить ещё до 60 секунд. Поэтому TTL кэша и ограничен 60 секундами, а не сроком сессии. Пока Redis недоступен, такой проблемы нет: все чтения идут в базу. +- Недоверие к кэшу после сбоя Redis живёт внутри процесса. С несколькими репликами API (lab3) реплика, которая сама не видела ошибку Redis, может прочитать старую копию отозванной сессии, пока метку не допишет другая реплика (обычно на первом же её запросе) или пока не истечёт TTL копии (до 60 с). Для реплик нужен общий признак, например номер поколения кэша в PostgreSQL. - Неизвестный токен при лежащей базе получает 503, а не 401. Проверить его без базы нельзя, а пустить нельзя, так что это осознанно. - PostgreSQL один, без реплик и бэкапов. Если пропадёт volume, пропадут данные. Durability — стандартная для PostgreSQL (`fsync`, `synchronous_commit` по умолчанию включены): после ответа 201 запись уже в WAL на диске. - Просроченные сессии из таблицы `sessions` не удаляются, они только перестают проходить проверку. Нужна периодическая чистка, сейчас её нет. diff --git a/evidence/gradle-tests.txt b/evidence/gradle-tests.txt index bd16314f..07f856b0 100644 --- a/evidence/gradle-tests.txt +++ b/evidence/gradle-tests.txt @@ -1,7 +1,7 @@ # ./gradlew test (Testcontainers: postgres:17.6-alpine3.22, redis:8.4.0-alpine) -total: 56 tests, 0 failed +total: 58 tests, 0 failed -infrastructure.controllers.authentication.AuthenticationControllerTest: 9 tests, 0 failed, 7.771 s +infrastructure.controllers.authentication.AuthenticationControllerTest: 9 tests, 0 failed, 4.301 s - logout revokes only the current token - Bearer token cannot open a session - password containing a colon and non-ASCII characters works @@ -11,19 +11,19 @@ infrastructure.controllers.authentication.AuthenticationControllerTest: 9 tests, - valid Basic credentials open a session - missing or malformed Basic header returns 401 - wrong password and unknown user get the same 401 -infrastructure.controllers.chat.ChatControllerTest: 6 tests, 0 failed, 2.042 s +infrastructure.controllers.chat.ChatControllerTest: 6 tests, 0 failed, 2.154 s - member can read own chat and members keep their order - unknown member is rejected before anything is written - foreign chat returns 404 for non-member - cursor issued to another user is rejected - chat list contains only own chats and is paginated - invalid member lists are rejected with 400 -infrastructure.controllers.health.HealthControllerTest: 2 tests, 0 failed, 0.128 s +infrastructure.controllers.health.HealthControllerTest: 2 tests, 0 failed, 0.125 s - live returns 200 - ready returns 200 when database is reachable -infrastructure.controllers.message.MessageConcurrencyTest: 1 tests, 0 failed, 0.281 s +infrastructure.controllers.message.MessageConcurrencyTest: 1 tests, 0 failed, 0.308 s - parallel senders produce exactly N unique messages -infrastructure.controllers.message.MessageControllerTest: 7 tests, 0 failed, 1.864 s +infrastructure.controllers.message.MessageControllerTest: 7 tests, 0 failed, 1.846 s - foreign chat returns 404 when sending - last page has null next_cursor - pagination walks all messages exactly once @@ -31,7 +31,7 @@ infrastructure.controllers.message.MessageControllerTest: 7 tests, 0 failed, 1.8 - empty text returns 400 invalid_request - history that is an exact multiple of limit ends with null next_cursor - foreign cursor returns 400 -infrastructure.controllers.user.UserControllerTest: 11 tests, 0 failed, 1.7 s +infrastructure.controllers.user.UserControllerTest: 11 tests, 0 failed, 1.861 s - password shorter than 12 code points returns 400 - concurrent registrations of one username create exactly one user - X-User-Id alone gives no access @@ -43,30 +43,32 @@ infrastructure.controllers.user.UserControllerTest: 11 tests, 0 failed, 1.7 s - Basic credentials are rejected on business endpoints - me returns the session owner - X-User-Id does not override bearer identity -infrastructure.repositories.authentication.session.cache.RedisSessionCacheRepositoryTest: 3 tests, 0 failed, 0.1 s +infrastructure.repositories.authentication.session.cache.RedisSessionCacheRepositoryTest: 3 tests, 0 failed, 0.141 s - cached session round-trips and expires together with the session - revoke replaces a cached session with a tombstone - late cache fill cannot overwrite a tombstone -infrastructure.repositories.chat.ExposedChatRepositoryTest: 2 tests, 0 failed, 0.529 s +infrastructure.repositories.chat.ExposedChatRepositoryTest: 2 tests, 0 failed, 0.479 s - chat row is rolled back when inserting members fails - successful creation commits the chat and every member -infrastructure.services.authentication.SessionLookupSingleFlightTest: 4 tests, 0 failed, 1.107 s +infrastructure.services.authentication.SessionLookupSingleFlightTest: 4 tests, 0 failed, 1.102 s - request arriving after logout does not join a lookup started before it - without single-flight every concurrent miss queries the database - with single-flight concurrent misses share one database lookup - lookup that finished before a request arrived is not reused -infrastructure.services.hashing.DefaultHashingServiceTest: 4 tests, 0 failed, 0.425 s +infrastructure.services.hashing.DefaultHashingServiceTest: 4 tests, 0 failed, 0.426 s - passwords differing only after 72 bytes are distinguished - validates the right password and rejects a wrong one - malformed hash is rejected instead of throwing - hash is a salted Argon2id PHC string -plugins.DatabaseMigrationTest: 1 tests, 0 failed, 0.206 s +plugins.DatabaseMigrationTest: 1 tests, 0 failed, 0.182 s - re-running migrations applies nothing and keeps data -plugins.DependencyFailureTest: 3 tests, 0 failed, 8.502 s +plugins.DependencyFailureTest: 5 tests, 0 failed, 25.189 s - without PostgreSQL requests fail with a bounded 503 + - logout during a Redis outage survives an API restart + - logout during a Redis outage is not undone by a stale cached copy - without Redis requests bypass the cache - hanging Redis costs one command timeout and recovers -plugins.SessionCacheTest: 3 tests, 0 failed, 0.963 s +plugins.SessionCacheTest: 3 tests, 0 failed, 1.025 s - emptied cache falls back to PostgreSQL - first authenticated request fills the cache - logout leaves a tombstone and the token is rejected diff --git a/evidence/lost-tombstone.txt b/evidence/lost-tombstone.txt new file mode 100644 index 00000000..47c8c12e --- /dev/null +++ b/evidence/lost-tombstone.txt @@ -0,0 +1,6 @@ +me: 200 +logout (redis paused): 204 +redis before restart: {"id":"3aa6c510-992e-45ed-9bb3-61f04e23d +me after api restart: 401 +redis after: revoked +mtusi-lab2-api | 2026-10-09T20:53:02.205Z INFO [DefaultDispatcher-worker-1] c.m.m.m.i.s.a.DefaultAuthenticationService - Wrote 1 pending session revocations to the cache diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/authentication/SessionRevocation.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/authentication/SessionRevocation.kt new file mode 100644 index 00000000..a02cf062 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/dto/authentication/SessionRevocation.kt @@ -0,0 +1,8 @@ +package com.mtusi.messenger.main.infrastructure.dto.authentication + +import kotlin.time.Instant + +data class SessionRevocation( + val tokenHash: String, + val expiresAt: Instant, +) diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/authentication/SessionRevocationModel.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/authentication/SessionRevocationModel.kt new file mode 100644 index 00000000..13e791dc --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/models/authentication/SessionRevocationModel.kt @@ -0,0 +1,11 @@ +package com.mtusi.messenger.main.infrastructure.models.authentication + +import org.jetbrains.exposed.v1.core.Table +import org.jetbrains.exposed.v1.datetime.timestamp + +object SessionRevocationModel : Table("session_revocations") { + val tokenHash = varchar("token_hash", 64) + val expiresAt = timestamp("expires_at") + + override val primaryKey = PrimaryKey(tokenHash, name = "session_revocations_pkey") +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/cache/RedisSessionCacheRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/cache/RedisSessionCacheRepository.kt index 2035dd4a..5cd22fc3 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/cache/RedisSessionCacheRepository.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/cache/RedisSessionCacheRepository.kt @@ -45,12 +45,13 @@ class RedisSessionCacheRepository( } override suspend fun revoke( - session: Session + tokenHash: String, + expiresAt: Instant ) { redis.commands().set( - key(session.tokenHash), + key(tokenHash), REVOKED, - SetArgs.Builder.pxAt(session.expiresAt.toEpochMilliseconds()) + SetArgs.Builder.pxAt(expiresAt.toEpochMilliseconds()) ).await() } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/cache/SessionCacheRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/cache/SessionCacheRepository.kt index c35340c1..c244cc95 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/cache/SessionCacheRepository.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/cache/SessionCacheRepository.kt @@ -24,10 +24,11 @@ interface SessionCacheRepository : Repository { ) /** - * Replaces the entry with a tombstone that lives until the session would have expired. + * Replaces the entry with a tombstone that lives until [expiresAt], when the session would have expired. */ suspend fun revoke( - session: Session + tokenHash: String, + expiresAt: Instant ) } diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/revocation/ExposedSessionRevocationRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/revocation/ExposedSessionRevocationRepository.kt new file mode 100644 index 00000000..9ca4227a --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/revocation/ExposedSessionRevocationRepository.kt @@ -0,0 +1,52 @@ +package com.mtusi.messenger.main.infrastructure.repositories.authentication.session.revocation + +import com.mtusi.messenger.main.infrastructure.dto.authentication.SessionRevocation +import com.mtusi.messenger.main.infrastructure.models.authentication.SessionRevocationModel +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.greater +import org.jetbrains.exposed.v1.core.lessEq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.koin.core.annotation.Single +import kotlin.time.Instant + +@Single +class ExposedSessionRevocationRepository : SessionRevocationRepository { + + override fun add( + revocation: SessionRevocation + ) { + SessionRevocationModel.insertIgnore { + it[tokenHash] = revocation.tokenHash + it[expiresAt] = revocation.expiresAt + } + } + + override fun findPending( + now: Instant + ): List { + return SessionRevocationModel + .selectAll() + .where(SessionRevocationModel.expiresAt greater now) + .map { + SessionRevocation( + tokenHash = it[SessionRevocationModel.tokenHash], + expiresAt = it[SessionRevocationModel.expiresAt] + ) + } + } + + override fun delete( + tokenHash: String + ) { + SessionRevocationModel.deleteWhere { SessionRevocationModel.tokenHash eq tokenHash } + } + + override fun deleteExpired( + now: Instant + ) { + SessionRevocationModel.deleteWhere { SessionRevocationModel.expiresAt lessEq now } + } + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/revocation/SessionRevocationRepository.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/revocation/SessionRevocationRepository.kt new file mode 100644 index 00000000..8612e876 --- /dev/null +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/revocation/SessionRevocationRepository.kt @@ -0,0 +1,31 @@ +package com.mtusi.messenger.main.infrastructure.repositories.authentication.session.revocation + +import com.mtusi.messenger.main.infrastructure.dto.authentication.SessionRevocation +import com.mtusi.messenger.main.infrastructure.repositories.Repository +import kotlin.time.Instant + +/** + * Logouts that still have to be written to the session cache as tombstones (outbox). + */ +interface SessionRevocationRepository : Repository { + + fun add( + revocation: SessionRevocation + ) + + /** + * Revocations of sessions that have not expired yet: an expired session is rejected even if its copy is cached. + */ + fun findPending( + now: Instant + ): List + + fun delete( + tokenHash: String + ) + + fun deleteExpired( + now: Instant + ) + +} diff --git a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/authentication/DefaultAuthenticationService.kt b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/authentication/DefaultAuthenticationService.kt index 8c4f6e59..9bfc3df9 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/authentication/DefaultAuthenticationService.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/infrastructure/services/authentication/DefaultAuthenticationService.kt @@ -4,17 +4,21 @@ import com.mtusi.messenger.main.config.authentication.AuthenticationConfig import com.mtusi.messenger.main.infrastructure.dto.User import com.mtusi.messenger.main.infrastructure.dto.authentication.IssuedSession import com.mtusi.messenger.main.infrastructure.dto.authentication.Session +import com.mtusi.messenger.main.infrastructure.dto.authentication.SessionRevocation import com.mtusi.messenger.main.infrastructure.repositories.authentication.session.SessionRepository import com.mtusi.messenger.main.infrastructure.repositories.authentication.session.cache.CachedSession import com.mtusi.messenger.main.infrastructure.repositories.authentication.session.cache.SessionCacheRepository +import com.mtusi.messenger.main.infrastructure.repositories.authentication.session.revocation.SessionRevocationRepository import com.mtusi.messenger.main.infrastructure.repositories.user.UserRepository import com.mtusi.messenger.main.infrastructure.services.hashing.HashingService import com.mtusi.messenger.main.infrastructure.services.hashing.types.HashingAlgorithm import com.mtusi.messenger.main.plugins.suspendedTransaction import com.mtusi.messenger.main.utils.computeSha256 import io.lettuce.core.RedisException +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.withContext import org.koin.core.annotation.Single import org.slf4j.LoggerFactory @@ -24,17 +28,22 @@ import java.util.Base64 import java.util.UUID import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicLong import kotlin.time.Clock import kotlin.time.Instant import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeSource private const val TOKEN_BYTES = 32 +private val CACHE_RESTORE_BACKOFF = 1.seconds + @Single class DefaultAuthenticationService( private val userRepository: UserRepository, private val sessionRepository: SessionRepository, private val sessionCacheRepository: SessionCacheRepository, + private val sessionRevocationRepository: SessionRevocationRepository, private val hashingService: HashingService, private val config: AuthenticationConfig ) : AuthenticationService { @@ -54,6 +63,21 @@ class DefaultAuthenticationService( */ private val isCacheAvailable = AtomicBoolean(true) + /** + * Number of failed cache operations. A failure may have lost a logout tombstone, so the cache is trusted only + * after the pending revocations were written to it with no failure since: [trustedAtFailures] equals this counter. + * A fresh process starts untrusted, as the previous one may have stopped with revocations still pending. + */ + private val cacheFailures = AtomicLong() + + @Volatile + private var trustedAtFailures = -1L + + private val cacheRestoreMutex = Mutex() + + @Volatile + private var nextCacheRestoreAttempt = TimeSource.Monotonic.markNow() + private val dummyPasswordHash by lazy { hashingService.computeHash(HashingAlgorithm.Argon2id, "dummy-password-for-timing") } @@ -155,14 +179,20 @@ class DefaultAuthenticationService( session: Session ): Boolean { + // The revocation is kept in PostgreSQL until its tombstone is in Redis: if Redis is unavailable now, + // the tombstone is written when the cache is trusted again (see restoreCache), even after a restart. val isDeleted = suspendedTransaction { + sessionRevocationRepository.add(SessionRevocation(session.tokenHash, session.expiresAt)) sessionRepository.deleteById(session.id) } // A lookup that started before the delete must not be joined by requests arriving after the logout. inFlightLookups.remove(session.tokenHash) - withCache { sessionCacheRepository.revoke(session) } + if (withCache { sessionCacheRepository.revoke(session.tokenHash, session.expiresAt) } != null) { + // Best effort: a leftover row only costs one more tombstone write on the next restore. + runCatchingDatabase { sessionRevocationRepository.delete(session.tokenHash) } + } return isDeleted } @@ -170,9 +200,71 @@ class DefaultAuthenticationService( private suspend fun findCachedSession( tokenHash: String ): CachedSession? { + if (trustedAtFailures != cacheFailures.get() && !restoreCache()) { + return null + } return withCache { sessionCacheRepository.find(tokenHash) } } + /** + * Writes the tombstones of logouts that happened while Redis was unavailable. Until that succeeds the cache + * is bypassed: a copy cached before the outage could otherwise outlive the logout. One request restores, + * the others go to the database meanwhile; a failed attempt is repeated after [CACHE_RESTORE_BACKOFF]. + */ + private suspend fun restoreCache(): Boolean { + + if (!cacheRestoreMutex.tryLock()) { + return false + } + + try { + val failures = cacheFailures.get() + + if (trustedAtFailures == failures) { + return true + } + if (nextCacheRestoreAttempt.hasNotPassedNow()) { + return false + } + nextCacheRestoreAttempt = TimeSource.Monotonic.markNow() + CACHE_RESTORE_BACKOFF + + val now = Clock.System.now() + + val pending = runCatchingDatabase { + sessionRevocationRepository.deleteExpired(now) + sessionRevocationRepository.findPending(now) + } ?: return false + + for (revocation in pending) { + withCache { sessionCacheRepository.revoke(revocation.tokenHash, revocation.expiresAt) } ?: return false + } + + if (pending.isNotEmpty()) { + runCatchingDatabase { pending.forEach { sessionRevocationRepository.delete(it.tokenHash) } } + logger.info("Wrote ${pending.size} pending session revocations to the cache") + } + + // A failure while restoring changed the counter, so the cache stays untrusted. + trustedAtFailures = failures + return trustedAtFailures == cacheFailures.get() + } finally { + cacheRestoreMutex.unlock() + } + } + + private suspend fun runCatchingDatabase( + statement: () -> T + ): T? { + return try { + suspendedTransaction { statement() } + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + logger.warn("Unable to update pending session revocations: ${e.message}") + null + } + } + private suspend fun cacheSession( session: Session, now: Instant @@ -209,6 +301,7 @@ class DefaultAuthenticationService( private fun onCacheUnavailable( cause: Exception ): Nothing? { + cacheFailures.incrementAndGet() if (isCacheAvailable.compareAndSet(true, false)) { logger.warn("Session cache is unavailable, sessions are read from the database: ${cause.message}") } diff --git a/src/main/kotlin/com/mtusi/messenger/main/plugins/Database.kt b/src/main/kotlin/com/mtusi/messenger/main/plugins/Database.kt index f3ea24ed..32001ed3 100644 --- a/src/main/kotlin/com/mtusi/messenger/main/plugins/Database.kt +++ b/src/main/kotlin/com/mtusi/messenger/main/plugins/Database.kt @@ -3,6 +3,7 @@ package com.mtusi.messenger.main.plugins import com.mtusi.messenger.main.config.database.DatabaseConfig import com.mtusi.messenger.main.config.database.SerialDatabaseConfig import com.mtusi.messenger.main.infrastructure.models.authentication.SessionModel +import com.mtusi.messenger.main.infrastructure.models.authentication.SessionRevocationModel import com.mtusi.messenger.main.infrastructure.models.chat.ChatMemberModel import com.mtusi.messenger.main.infrastructure.models.chat.ChatModel import com.mtusi.messenger.main.infrastructure.models.message.MessageModel @@ -95,7 +96,8 @@ private fun generateMigration(description: String) { ChatModel, ChatMemberModel, MessageModel, - SessionModel + SessionModel, + SessionRevocationModel ) val config = HoconApplicationConfig(ConfigFactory.parseFile(File(DEVELOPMENT_DATABASE_CONFIG)).resolve()) diff --git a/src/main/resources/migrations/V2__session_revocations.sql b/src/main/resources/migrations/V2__session_revocations.sql new file mode 100644 index 00000000..4b15c00c --- /dev/null +++ b/src/main/resources/migrations/V2__session_revocations.sql @@ -0,0 +1,6 @@ +-- Logouts whose tombstone has not reached Redis yet. Written in the same transaction as the session delete and +-- removed once the tombstone is stored, so a copy cached before an outage cannot outlive the logout. +CREATE TABLE session_revocations ( + token_hash varchar(64) PRIMARY KEY, + expires_at timestamptz NOT NULL +); diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/cache/RedisSessionCacheRepositoryTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/cache/RedisSessionCacheRepositoryTest.kt index ed2007dc..8f27e66f 100644 --- a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/cache/RedisSessionCacheRepositoryTest.kt +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/repositories/authentication/session/cache/RedisSessionCacheRepositoryTest.kt @@ -63,7 +63,7 @@ class RedisSessionCacheRepositoryTest { val session = session() repository.putIfAbsent(session, cachedUntil = session.expiresAt) - repository.revoke(session) + repository.revoke(session.tokenHash, session.expiresAt) assertEquals(CachedSession.Revoked, repository.find(session.tokenHash)) assertTrue(ttlSeconds(session) in 590..600, "tombstone must live until the session would expire") @@ -74,7 +74,7 @@ class RedisSessionCacheRepositoryTest { val session = session() // A lookup read the session from PostgreSQL before logout and stores it after the logout finished. - repository.revoke(session) + repository.revoke(session.tokenHash, session.expiresAt) repository.putIfAbsent(session, cachedUntil = session.expiresAt) assertEquals(CachedSession.Revoked, repository.find(session.tokenHash)) diff --git a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/authentication/SessionLookupSingleFlightTest.kt b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/authentication/SessionLookupSingleFlightTest.kt index 06791e4c..01df4eed 100644 --- a/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/authentication/SessionLookupSingleFlightTest.kt +++ b/src/test/kotlin/com/mtusi/messenger/main/infrastructure/services/authentication/SessionLookupSingleFlightTest.kt @@ -2,9 +2,11 @@ package com.mtusi.messenger.main.infrastructure.services.authentication import com.mtusi.messenger.main.config.authentication.SerialAuthenticationConfig import com.mtusi.messenger.main.infrastructure.dto.authentication.Session +import com.mtusi.messenger.main.infrastructure.dto.authentication.SessionRevocation import com.mtusi.messenger.main.infrastructure.repositories.authentication.session.SessionRepository import com.mtusi.messenger.main.infrastructure.repositories.authentication.session.cache.CachedSession import com.mtusi.messenger.main.infrastructure.repositories.authentication.session.cache.SessionCacheRepository +import com.mtusi.messenger.main.infrastructure.repositories.authentication.session.revocation.SessionRevocationRepository import com.mtusi.messenger.main.infrastructure.repositories.user.ExposedUserRepository import com.mtusi.messenger.main.infrastructure.services.hashing.DefaultHashingService import com.mtusi.messenger.main.support.TestDatabase @@ -74,7 +76,19 @@ class SessionLookupSingleFlightTest { writes.incrementAndGet() } - override suspend fun revoke(session: Session) = Unit + override suspend fun revoke(tokenHash: String, expiresAt: Instant) = Unit + + } + + private object NoRevocations : SessionRevocationRepository { + + override fun add(revocation: SessionRevocation) = Unit + + override fun findPending(now: Instant) = emptyList() + + override fun delete(tokenHash: String) = Unit + + override fun deleteExpired(now: Instant) = Unit } @@ -89,6 +103,7 @@ class SessionLookupSingleFlightTest { userRepository = ExposedUserRepository(), sessionRepository = repository, sessionCacheRepository = cache, + sessionRevocationRepository = NoRevocations, hashingService = DefaultHashingService(), config = SerialAuthenticationConfig( realm = "messenger", diff --git a/src/test/kotlin/com/mtusi/messenger/main/plugins/DependencyFailureTest.kt b/src/test/kotlin/com/mtusi/messenger/main/plugins/DependencyFailureTest.kt index eaec7e10..2a171457 100644 --- a/src/test/kotlin/com/mtusi/messenger/main/plugins/DependencyFailureTest.kt +++ b/src/test/kotlin/com/mtusi/messenger/main/plugins/DependencyFailureTest.kt @@ -1,8 +1,11 @@ package com.mtusi.messenger.main.plugins import com.mtusi.messenger.main.extensions.ErrorBody +import com.mtusi.messenger.main.infrastructure.controllers.message.requests.CreateMessageRequest +import com.mtusi.messenger.main.infrastructure.dto.Chat import com.mtusi.messenger.main.support.TestDatabase import com.mtusi.messenger.main.support.TestRedis +import com.mtusi.messenger.main.support.TestUser import com.mtusi.messenger.main.support.auth import com.mtusi.messenger.main.support.createChat import com.mtusi.messenger.main.support.register @@ -15,8 +18,12 @@ import io.ktor.client.request.delete import io.ktor.client.request.get import io.ktor.client.request.header import io.ktor.client.request.post +import io.ktor.client.request.setBody import io.ktor.client.statement.HttpResponse +import io.ktor.http.ContentType import io.ktor.http.HttpStatusCode +import io.ktor.http.contentType +import org.testcontainers.containers.GenericContainer import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -101,6 +108,84 @@ class DependencyFailureTest { } } + /** + * The tombstone written by a logout during an outage can be lost while the copy cached before it survives + * (here: put back by hand). The revocation kept in PostgreSQL must still win once Redis is reachable again. + */ + private fun loseTombstone(redis: GenericContainer<*>, user: TestUser, cachedCopy: String) { + TestRedis.cli(redis, "SET", "session:${user.token.toByteArray().computeSha256()}", cachedCopy, "PX", "60000") + } + + private fun cachedCopy(redis: GenericContainer<*>, user: TestUser): String { + return TestRedis.cli(redis, "GET", "session:${user.token.toByteArray().computeSha256()}") + .also { assertTrue(it.contains(user.id.toString()), "session must be cached before the outage") } + } + + @Test + fun `logout during a Redis outage is not undone by a stale cached copy`() { + val redis = TestRedis.create().also { it.start() } + val docker = redis.dockerClient + + try { + runApp(redis = TestRedis.endpoint(redis)) { client -> + val alice = client.register("alice") + assertEquals(HttpStatusCode.OK, client.get("/api/v1/users/me") { auth(alice) }.status) + val copy = cachedCopy(redis, alice) + + docker.pauseContainerCmd(redis.containerId).exec() + assertEquals(HttpStatusCode.NoContent, client.delete("/api/v1/auth/sessions/current") { auth(alice) }.status) + docker.unpauseContainerCmd(redis.containerId).exec() + + loseTombstone(redis, alice, copy) + + assertEquals(HttpStatusCode.Unauthorized, client.get("/api/v1/users/me") { auth(alice) }.status) + assertEquals("revoked", TestRedis.cli(redis, "GET", "session:${alice.token.toByteArray().computeSha256()}")) + } + } finally { + redis.stop() + } + } + + @Test + fun `logout during a Redis outage survives an API restart`() { + val redis = TestRedis.create().also { it.start() } + val docker = redis.dockerClient + + lateinit var alice: TestUser + lateinit var chat: Chat + lateinit var copy: String + + try { + runApp(redis = TestRedis.endpoint(redis)) { client -> + alice = client.register("alice") + val bob = client.register("bob") + chat = client.createChat(alice, "restart", listOf(alice.id, bob.id)) + copy = cachedCopy(redis, alice) + + docker.pauseContainerCmd(redis.containerId).exec() + assertEquals(HttpStatusCode.NoContent, client.delete("/api/v1/auth/sessions/current") { auth(alice) }.status) + } + + docker.unpauseContainerCmd(redis.containerId).exec() + loseTombstone(redis, alice, copy) + + runApp(redis = TestRedis.endpoint(redis)) { client -> + assertEquals(HttpStatusCode.Unauthorized, client.get("/api/v1/users/me") { auth(alice) }.status) + assertEquals(HttpStatusCode.Unauthorized, client.get("/api/v1/chats/${chat.id}/messages") { auth(alice) }.status) + assertEquals( + HttpStatusCode.Unauthorized, + client.post("/api/v1/chats/${chat.id}/messages") { + auth(alice) + contentType(ContentType.Application.Json) + setBody(CreateMessageRequest(text = "after logout")) + }.status + ) + } + } finally { + redis.stop() + } + } + @Test fun `without PostgreSQL requests fail with a bounded 503`() { val database = TestDatabase.create().also { it.start() }