forked from mtusi2026/mtusi2026lab2
restore Redis cache after outage
This commit is contained in:
@@ -86,7 +86,9 @@ Redis кэширует только первую страницу истории
|
||||
`limit` и `history_version` из PostgreSQL. В той же транзакции, что и
|
||||
вставка сообщения, PostgreSQL увеличивает `history_version`; старый ключ
|
||||
поэтому больше не читается. Redis не источник истины: при его отказе кэш
|
||||
отключается в процессе API, а чтение продолжает PostgreSQL. Redis использует
|
||||
временно обходится, а чтение продолжает PostgreSQL. API повторно проверяет
|
||||
Redis не чаще одного раза в секунду; после успешного `PING` заново создаёт
|
||||
клиент, и кэш с Redis-lock снова используются без restart API. Redis использует
|
||||
AOF и named volume `redis_data`.
|
||||
|
||||
## Доказательства критериев
|
||||
@@ -103,7 +105,7 @@ AOF и named volume `redis_data`.
|
||||
| TTL и зависимости | `make test-grade4`; TTL=2 | Expired session → 401; Redis fallback проходит smoke; DB readiness → 503 | `evidence/grade4-checks-2026-10-08.txt` |
|
||||
| Индекс истории | `./scripts/explain_history.sh` | Index Scan, 50 строк, shared hit=3 | `evidence/explain-history-2026-10-08.txt` |
|
||||
| Трек 5: stampede | `student-tests/cache_stampede.py` | 20 SQL-загрузок → 1 при 20 читателях | `evidence/track1-cache-stampede-2026-10-08.txt` |
|
||||
| Трек 5: restart cache | `student-tests/cache_recovery.py` | История не изменилась после restart Redis | `evidence/track1-cache-stampede-2026-10-08.txt` |
|
||||
| Трек 5: отказ и восстановление cache | `student-tests/cache_recovery.py` | PostgreSQL fallback, затем кэш и lock восстановлены без restart API | `evidence/redis-recovery-2026-10-09.txt` |
|
||||
|
||||
## Трек 5 — согласованность кэша
|
||||
|
||||
@@ -132,8 +134,11 @@ API с `CACHE_STAMPEDE_PROTECTION=false`; затем тот же сценари
|
||||
|
||||
read-after-write проверяет `student-tests/lab4_checks.py`: страница сначала
|
||||
кэшируется, затем создаётся сообщение, и следующий GET содержит оба сообщения.
|
||||
`student-tests/cache_recovery.py` заполняет кэш, перезапускает Redis и
|
||||
проверяет неизменность ответа. Raw results и команды сохранены в evidence.
|
||||
`student-tests/cache_recovery.py` сначала останавливает Redis и подтверждает
|
||||
корректный PostgreSQL fallback. Затем Redis запускается обратно без restart API:
|
||||
тест проверяет создание нового ключа кэша и ровно одну SQL-загрузку для 20
|
||||
одновременных запросов новой версии истории. Raw results и команды сохранены в
|
||||
evidence.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
PASS: Redis cache and stampede lock recover without API restart
|
||||
@@ -1,5 +1,6 @@
|
||||
import json
|
||||
from asyncio import sleep
|
||||
from asyncio import Lock, sleep
|
||||
from time import monotonic
|
||||
from secrets import token_urlsafe
|
||||
from uuid import UUID
|
||||
|
||||
@@ -14,28 +15,55 @@ class HistoryCache:
|
||||
self._url = url
|
||||
self._ttl_seconds = ttl_seconds
|
||||
self._client: Redis | None = None
|
||||
self._disabled = not bool(url)
|
||||
self._reconnect_lock = Lock()
|
||||
self._next_reconnect_at = 0.0
|
||||
self._stampede_protection = stampede_protection
|
||||
|
||||
async def connect(self) -> None:
|
||||
if self._disabled:
|
||||
return
|
||||
try:
|
||||
client = Redis.from_url(
|
||||
self._url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=1,
|
||||
socket_timeout=1,
|
||||
max_connections=50,
|
||||
)
|
||||
await client.ping()
|
||||
self._client = client
|
||||
except RedisError:
|
||||
self._disabled = True
|
||||
await self._client_or_none()
|
||||
|
||||
async def _client_or_none(self) -> Redis | None:
|
||||
"""Возвращает рабочий клиент или временно обходит кэш при отказе Redis."""
|
||||
if not self._url:
|
||||
return None
|
||||
if self._client is not None:
|
||||
return self._client
|
||||
if monotonic() < self._next_reconnect_at:
|
||||
return None
|
||||
|
||||
async with self._reconnect_lock:
|
||||
if self._client is not None:
|
||||
return self._client
|
||||
if monotonic() < self._next_reconnect_at:
|
||||
return None
|
||||
try:
|
||||
client = Redis.from_url(
|
||||
self._url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=1,
|
||||
socket_timeout=1,
|
||||
max_connections=50,
|
||||
)
|
||||
await client.ping()
|
||||
self._client = client
|
||||
return client
|
||||
except RedisError:
|
||||
self._next_reconnect_at = monotonic() + 1
|
||||
return None
|
||||
|
||||
async def _mark_unavailable(self, client: Redis) -> None:
|
||||
"""Отбрасывает сломанное соединение и разрешает повторное подключение."""
|
||||
async with self._reconnect_lock:
|
||||
if self._client is client:
|
||||
self._client = None
|
||||
self._next_reconnect_at = monotonic() + 1
|
||||
await client.aclose()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._client is not None:
|
||||
await self._client.aclose()
|
||||
if self._client is None:
|
||||
return
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
@staticmethod
|
||||
def _key(chat_id: UUID, version: int, limit: int) -> str:
|
||||
@@ -46,39 +74,44 @@ class HistoryCache:
|
||||
return cls._key(chat_id, version, limit) + ":lock"
|
||||
|
||||
async def get(self, chat_id: UUID, version: int, limit: int) -> list[dict] | None:
|
||||
if self._disabled or self._client is None:
|
||||
client = await self._client_or_none()
|
||||
if client is None:
|
||||
return None
|
||||
try:
|
||||
value = await self._client.get(self._key(chat_id, version, limit))
|
||||
value = await client.get(self._key(chat_id, version, limit))
|
||||
return json.loads(value) if value else None
|
||||
except (RedisError, ValueError, TypeError):
|
||||
self._disabled = True
|
||||
except RedisError:
|
||||
await self._mark_unavailable(client)
|
||||
return None
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
async def set(self, chat_id: UUID, version: int, limit: int, items: list[dict]) -> None:
|
||||
if self._disabled or self._client is None:
|
||||
client = await self._client_or_none()
|
||||
if client is None:
|
||||
return
|
||||
try:
|
||||
encoded = json.dumps(items, default=str, separators=(",", ":"))
|
||||
await self._client.set(
|
||||
await client.set(
|
||||
self._key(chat_id, version, limit), encoded, ex=self._ttl_seconds
|
||||
)
|
||||
except RedisError:
|
||||
self._disabled = True
|
||||
await self._mark_unavailable(client)
|
||||
|
||||
async def acquire_load_lock(
|
||||
self, chat_id: UUID, version: int, limit: int
|
||||
) -> str | None:
|
||||
if self._disabled or self._client is None or not self._stampede_protection:
|
||||
client = await self._client_or_none()
|
||||
if client is None or not self._stampede_protection:
|
||||
return None
|
||||
token = token_urlsafe(16)
|
||||
try:
|
||||
locked = await self._client.set(
|
||||
locked = await client.set(
|
||||
self._lock_key(chat_id, version, limit), token, nx=True, ex=2
|
||||
)
|
||||
return token if locked else ""
|
||||
except RedisError:
|
||||
self._disabled = True
|
||||
await self._mark_unavailable(client)
|
||||
return None
|
||||
|
||||
async def wait_for_fill(
|
||||
@@ -87,21 +120,22 @@ class HistoryCache:
|
||||
for _ in range(20):
|
||||
await sleep(0.05)
|
||||
cached = await self.get(chat_id, version, limit)
|
||||
if cached is not None or self._disabled:
|
||||
if cached is not None or self._client is None:
|
||||
return cached
|
||||
return None
|
||||
|
||||
async def release_load_lock(
|
||||
self, chat_id: UUID, version: int, limit: int, token: str
|
||||
) -> None:
|
||||
if self._disabled or self._client is None:
|
||||
client = await self._client_or_none()
|
||||
if client is None:
|
||||
return
|
||||
try:
|
||||
await self._client.eval(
|
||||
await client.eval(
|
||||
"if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) end return 0",
|
||||
1,
|
||||
self._lock_key(chat_id, version, limit),
|
||||
token,
|
||||
)
|
||||
except RedisError:
|
||||
self._disabled = True
|
||||
await self._mark_unavailable(client)
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Доказывает корректное чтение после restart Redis."""
|
||||
"""Проверяет fallback и восстановление Redis-кэша без restart API."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
@@ -19,11 +22,47 @@ def main() -> None:
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
|
||||
before = client.call("GET", path, actor=alice, schema="MessagePage")["json"]
|
||||
check(before["items"] == [message], "initial cache fill failed")
|
||||
subprocess.run(["docker", "compose", "restart", "redis"], cwd=ROOT, check=True)
|
||||
time.sleep(1)
|
||||
|
||||
subprocess.run(["docker", "compose", "stop", "redis"], cwd=ROOT, check=True)
|
||||
fallback = client.call("GET", path, actor=alice, schema="MessagePage")["json"]
|
||||
check(fallback["items"] == [message], "PostgreSQL fallback returned a different history")
|
||||
|
||||
subprocess.run(["docker", "compose", "start", "redis"], cwd=ROOT, check=True)
|
||||
time.sleep(1.1) # Интервал повторного подключения к Redis в приложении.
|
||||
after = client.call("GET", path, actor=alice, schema="MessagePage")["json"]
|
||||
check(after["items"] == [message], "history changed after Redis restart")
|
||||
print("PASS: history remains correct after Redis restart")
|
||||
|
||||
keys = subprocess.check_output(
|
||||
[
|
||||
"docker", "compose", "exec", "-T", "redis", "redis-cli", "--scan",
|
||||
"--pattern", "history:first:" + chat["id"] + ":*",
|
||||
],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
)
|
||||
check("history:first:" + chat["id"] in keys, "cache was not restored after Redis restart")
|
||||
|
||||
second = client.message(alice, chat, "cold page after Redis recovery")["json"]
|
||||
started_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
workers = 20
|
||||
barrier = threading.Barrier(workers)
|
||||
|
||||
def read_cold_page(_worker: int) -> None:
|
||||
barrier.wait()
|
||||
response = Client().call("GET", path, actor=alice, schema="MessagePage")["json"]
|
||||
check(response["items"] == [message, second], "concurrent history differs")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
list(pool.map(read_cold_page, range(workers)))
|
||||
|
||||
logs = subprocess.check_output(
|
||||
["docker", "compose", "logs", "--since", started_at, "api"],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
)
|
||||
marker = "history_sql_page_load chat_id=" + chat["id"] + " version=2 limit=100"
|
||||
check(logs.count(marker) == 1, "Redis lock was not restored after Redis restart")
|
||||
print("PASS: Redis cache and stampede lock recover without API restart")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user