diff --git a/REPORT.md b/REPORT.md index 901791a9..8673f3df 100644 --- a/REPORT.md +++ b/REPORT.md @@ -86,7 +86,9 @@ Redis кэширует только первую страницу истории `limit` и `history_version` из PostgreSQL. В той же транзакции, что и вставка сообщения, PostgreSQL увеличивает `history_version`; старый ключ поэтому больше не читается. Redis не источник истины: при его отказе кэш -отключается в процессе API, а чтение продолжает PostgreSQL. Redis использует +временно обходится, а чтение продолжает PostgreSQL. API повторно проверяет +Redis не чаще одного раза в секунду; после успешного `PING` заново создаёт +клиент, и кэш с Redis-lock снова используются без restart API. Redis использует AOF и named volume `redis_data`. ## Доказательства критериев @@ -103,7 +105,7 @@ AOF и named volume `redis_data`. | TTL и зависимости | `make test-grade4`; TTL=2 | Expired session → 401; Redis fallback проходит smoke; DB readiness → 503 | `evidence/grade4-checks-2026-10-08.txt` | | Индекс истории | `./scripts/explain_history.sh` | Index Scan, 50 строк, shared hit=3 | `evidence/explain-history-2026-10-08.txt` | | Трек 5: stampede | `student-tests/cache_stampede.py` | 20 SQL-загрузок → 1 при 20 читателях | `evidence/track1-cache-stampede-2026-10-08.txt` | -| Трек 5: restart cache | `student-tests/cache_recovery.py` | История не изменилась после restart Redis | `evidence/track1-cache-stampede-2026-10-08.txt` | +| Трек 5: отказ и восстановление cache | `student-tests/cache_recovery.py` | PostgreSQL fallback, затем кэш и lock восстановлены без restart API | `evidence/redis-recovery-2026-10-09.txt` | ## Трек 5 — согласованность кэша @@ -132,8 +134,11 @@ API с `CACHE_STAMPEDE_PROTECTION=false`; затем тот же сценари read-after-write проверяет `student-tests/lab4_checks.py`: страница сначала кэшируется, затем создаётся сообщение, и следующий GET содержит оба сообщения. -`student-tests/cache_recovery.py` заполняет кэш, перезапускает Redis и -проверяет неизменность ответа. Raw results и команды сохранены в evidence. +`student-tests/cache_recovery.py` сначала останавливает Redis и подтверждает +корректный PostgreSQL fallback. Затем Redis запускается обратно без restart API: +тест проверяет создание нового ключа кэша и ровно одну SQL-загрузку для 20 +одновременных запросов новой версии истории. Raw results и команды сохранены в +evidence. ## Ограничения и следующие шаги diff --git a/evidence/redis-recovery-2026-10-09.txt b/evidence/redis-recovery-2026-10-09.txt new file mode 100644 index 00000000..1d29fb20 --- /dev/null +++ b/evidence/redis-recovery-2026-10-09.txt @@ -0,0 +1 @@ +PASS: Redis cache and stampede lock recover without API restart diff --git a/src/storage/history_cache.py b/src/storage/history_cache.py index 6e36c16f..616444f0 100644 --- a/src/storage/history_cache.py +++ b/src/storage/history_cache.py @@ -1,5 +1,6 @@ import json -from asyncio import sleep +from asyncio import Lock, sleep +from time import monotonic from secrets import token_urlsafe from uuid import UUID @@ -14,28 +15,55 @@ class HistoryCache: self._url = url self._ttl_seconds = ttl_seconds self._client: Redis | None = None - self._disabled = not bool(url) + self._reconnect_lock = Lock() + self._next_reconnect_at = 0.0 self._stampede_protection = stampede_protection async def connect(self) -> None: - if self._disabled: - return - try: - client = Redis.from_url( - self._url, - decode_responses=True, - socket_connect_timeout=1, - socket_timeout=1, - max_connections=50, - ) - await client.ping() - self._client = client - except RedisError: - self._disabled = True + await self._client_or_none() + + async def _client_or_none(self) -> Redis | None: + """Возвращает рабочий клиент или временно обходит кэш при отказе Redis.""" + if not self._url: + return None + if self._client is not None: + return self._client + if monotonic() < self._next_reconnect_at: + return None + + async with self._reconnect_lock: + if self._client is not None: + return self._client + if monotonic() < self._next_reconnect_at: + return None + try: + client = Redis.from_url( + self._url, + decode_responses=True, + socket_connect_timeout=1, + socket_timeout=1, + max_connections=50, + ) + await client.ping() + self._client = client + return client + except RedisError: + self._next_reconnect_at = monotonic() + 1 + return None + + async def _mark_unavailable(self, client: Redis) -> None: + """Отбрасывает сломанное соединение и разрешает повторное подключение.""" + async with self._reconnect_lock: + if self._client is client: + self._client = None + self._next_reconnect_at = monotonic() + 1 + await client.aclose() async def close(self) -> None: - if self._client is not None: - await self._client.aclose() + if self._client is None: + return + await self._client.aclose() + self._client = None @staticmethod def _key(chat_id: UUID, version: int, limit: int) -> str: @@ -46,39 +74,44 @@ class HistoryCache: return cls._key(chat_id, version, limit) + ":lock" async def get(self, chat_id: UUID, version: int, limit: int) -> list[dict] | None: - if self._disabled or self._client is None: + client = await self._client_or_none() + if client is None: return None try: - value = await self._client.get(self._key(chat_id, version, limit)) + value = await client.get(self._key(chat_id, version, limit)) return json.loads(value) if value else None - except (RedisError, ValueError, TypeError): - self._disabled = True + except RedisError: + await self._mark_unavailable(client) + return None + except (ValueError, TypeError): return None async def set(self, chat_id: UUID, version: int, limit: int, items: list[dict]) -> None: - if self._disabled or self._client is None: + client = await self._client_or_none() + if client is None: return try: encoded = json.dumps(items, default=str, separators=(",", ":")) - await self._client.set( + await client.set( self._key(chat_id, version, limit), encoded, ex=self._ttl_seconds ) except RedisError: - self._disabled = True + await self._mark_unavailable(client) async def acquire_load_lock( self, chat_id: UUID, version: int, limit: int ) -> str | None: - if self._disabled or self._client is None or not self._stampede_protection: + client = await self._client_or_none() + if client is None or not self._stampede_protection: return None token = token_urlsafe(16) try: - locked = await self._client.set( + locked = await client.set( self._lock_key(chat_id, version, limit), token, nx=True, ex=2 ) return token if locked else "" except RedisError: - self._disabled = True + await self._mark_unavailable(client) return None async def wait_for_fill( @@ -87,21 +120,22 @@ class HistoryCache: for _ in range(20): await sleep(0.05) cached = await self.get(chat_id, version, limit) - if cached is not None or self._disabled: + if cached is not None or self._client is None: return cached return None async def release_load_lock( self, chat_id: UUID, version: int, limit: int, token: str ) -> None: - if self._disabled or self._client is None: + client = await self._client_or_none() + if client is None: return try: - await self._client.eval( + await client.eval( "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) end return 0", 1, self._lock_key(chat_id, version, limit), token, ) except RedisError: - self._disabled = True + await self._mark_unavailable(client) diff --git a/student-tests/cache_recovery.py b/student-tests/cache_recovery.py index e5dbd86e..7571d5e0 100644 --- a/student-tests/cache_recovery.py +++ b/student-tests/cache_recovery.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 -"""Доказывает корректное чтение после restart Redis.""" +"""Проверяет fallback и восстановление Redis-кэша без restart API.""" +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone from pathlib import Path import subprocess import sys +import threading import time sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) @@ -19,11 +22,47 @@ def main() -> None: path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" before = client.call("GET", path, actor=alice, schema="MessagePage")["json"] check(before["items"] == [message], "initial cache fill failed") - subprocess.run(["docker", "compose", "restart", "redis"], cwd=ROOT, check=True) - time.sleep(1) + + subprocess.run(["docker", "compose", "stop", "redis"], cwd=ROOT, check=True) + fallback = client.call("GET", path, actor=alice, schema="MessagePage")["json"] + check(fallback["items"] == [message], "PostgreSQL fallback returned a different history") + + subprocess.run(["docker", "compose", "start", "redis"], cwd=ROOT, check=True) + time.sleep(1.1) # Интервал повторного подключения к Redis в приложении. after = client.call("GET", path, actor=alice, schema="MessagePage")["json"] check(after["items"] == [message], "history changed after Redis restart") - print("PASS: history remains correct after Redis restart") + + keys = subprocess.check_output( + [ + "docker", "compose", "exec", "-T", "redis", "redis-cli", "--scan", + "--pattern", "history:first:" + chat["id"] + ":*", + ], + cwd=ROOT, + text=True, + ) + check("history:first:" + chat["id"] in keys, "cache was not restored after Redis restart") + + second = client.message(alice, chat, "cold page after Redis recovery")["json"] + started_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + workers = 20 + barrier = threading.Barrier(workers) + + def read_cold_page(_worker: int) -> None: + barrier.wait() + response = Client().call("GET", path, actor=alice, schema="MessagePage")["json"] + check(response["items"] == [message, second], "concurrent history differs") + + with ThreadPoolExecutor(max_workers=workers) as pool: + list(pool.map(read_cold_page, range(workers))) + + logs = subprocess.check_output( + ["docker", "compose", "logs", "--since", started_at, "api"], + cwd=ROOT, + text=True, + ) + marker = "history_sql_page_load chat_id=" + chat["id"] + " version=2 limit=100" + check(logs.count(marker) == 1, "Redis lock was not restored after Redis restart") + print("PASS: Redis cache and stampede lock recover without API restart") if __name__ == "__main__":