Реализовал уровень 4: Redis-кэш, проверки отказов и индекс истории

This commit is contained in:
2026-10-09 14:19:48 +03:00
parent 6fba9410d5
commit 64f2a5b468
16 changed files with 321 additions and 8 deletions
+1
View File
@@ -2,3 +2,4 @@
*.py text eol=lf
*.sql text eol=lf
Dockerfile text eol=lf
*.cmd text eol=crlf
+7
View File
@@ -31,3 +31,10 @@ bootstrap:
student-test:
$(PYTHON) student-tests/test_lab2.py --base-url "$(BASE_URL)"
.PHONY: level4-test explain
level4-test:
$(PYTHON) student-tests/test_level4.py
explain:
$(PYTHON) student-tests/explain_level4.py
+14
View File
@@ -15,11 +15,24 @@ services:
retries: 30
cpus: 1
mem_limit: 512m
redis:
image: redis:7.4.1-alpine
command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"]
volumes: [redis_data:/data]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 3s
retries: 20
cpus: 0.5
mem_limit: 128m
api:
build: .
image: lyakhonov-messenger:lab2
environment:
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL: redis://redis:6379/0
PROFILE_CACHE_TTL_SECONDS: 60
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
secrets: [database_url]
ports: ["127.0.0.1:8080:8080"]
@@ -35,4 +48,5 @@ secrets:
database_url:
file: ./secrets/database_url
volumes:
redis_data:
postgres_data:
+76
View File
@@ -0,0 +1,76 @@
{
"rows": 10000,
"query": "SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT 51",
"plan": {
"Plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 0.29,
"Total Cost": 12.94,
"Plan Rows": 51,
"Plan Width": 87,
"Actual Startup Time": 0.017,
"Actual Total Time": 0.025,
"Actual Rows": 51,
"Actual Loops": 1,
"Shared Hit Blocks": 5,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "messages_history_idx",
"Relation Name": "messages",
"Alias": "messages",
"Startup Cost": 0.29,
"Total Cost": 536.25,
"Plan Rows": 2160,
"Plan Width": 87,
"Actual Startup Time": 0.016,
"Actual Total Time": 0.021,
"Actual Rows": 51,
"Actual Loops": 1,
"Index Cond": "((chat_id = '44cfe328-842b-43a5-900b-ff083cb13f7f'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.008001+00'::timestamp with time zone, '10e68164-4b26-406f-a3a4-f0395ca68f52'::uuid)))",
"Rows Removed by Index Recheck": 0,
"Shared Hit Blocks": 5,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
},
"Planning": {
"Shared Hit Blocks": 108,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
},
"Planning Time": 0.516,
"Triggers": [],
"Execution Time": 0.08
}
}
+22
View File
@@ -0,0 +1,22 @@
{
"registration_codes": [
201,
409,
409,
409,
409,
409,
409,
409
],
"rollback": "PASS",
"profile_cache_ttl_seconds": 59,
"redis_fallback": "PASS",
"redis_stop_and_request_seconds": 8.617,
"db_failure": "503, live=200, recovery with same session",
"two_db_error_requests_seconds": 6.05,
"expiry_seconds": 2,
"expiry": "401",
"logout": "401",
"passed": true
}
+2
View File
@@ -0,0 +1,2 @@
python tests/smoke.py
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
+3
View File
@@ -7,3 +7,6 @@
Индекс messages_history_idx (chat_id, created_at, id) обслуживает WHERE chat_id = ? AND (created_at,id) > (?,?) ORDER BY created_at,id LIMIT ?. Запрашиваются limit+1 строка, чтобы определить наличие следующей страницы. PostgreSQL timestamptz сохраняет микросекунды; UUID разрешает совпадения времени. Между страницами нет общего snapshot.
Миграции выполняются под advisory lock в транзакции; schema_migrations предотвращает повторное применение. Пул ограничен десятью подключениями и 32 ожидающими; получение подключения — 3 секунды, statement_timeout — 3 секунды, lock_timeout — 2 секунды.
## Кэш уровня 4
Redis хранит только публичные неизменяемые профили пользователей, без хешей паролей и токенов. Ключ profile:v1:UUID, TTL 60 секунд, maxmemory 64 MiB, allkeys-lru. Пользовательские изменения и удаление не входят в API: инвалидация при обновлении не требуется; при будущем расширении она обязательна. Cache-aside: промах или ошибка возвращают чтение PostgreSQL; кэш заполняется после чтения подтверждённой строки. После рестарта кэш пуст и восстанавливается по запросам. Persistence кэша отключён намеренно; сессии находятся в PostgreSQL. Redis не критичен для readiness, PostgreSQL критичен. Кэш не участвует в проверке Bearer: каждая сессия проверяется в PostgreSQL, поэтому logout немедленно действует даже при недоступном Redis.
+1
View File
@@ -22,3 +22,4 @@ uvicorn==0.32.0
uvloop==0.23.0
watchfiles==1.3.0
websockets==17.2
redis==5.2.0
+1
View File
@@ -4,3 +4,4 @@ pydantic==2.9.2
psycopg[binary]==3.2.3
psycopg-pool==3.2.4
argon2-cffi==23.1.0
redis==5.2.0
+3 -3
View File
@@ -1,3 +1,3 @@
@echo off␍
cd /d "%~dp0.."␍
docker compose restart db api␍
@echo off
cd /d "%~dp0.."
docker compose restart db redis api
+2 -2
View File
@@ -1,5 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Перезапускаем реальные процессы, сохраняя именованные тома.
docker compose restart db api
# Перезапускаем все процессы, не удаляя тома и не изменяя данные.
docker compose restart db redis api
+43
View File
@@ -0,0 +1,43 @@
"""Необязательный кэш неизменяемого публичного профиля; ошибки обходятся через БД."""
import logging
import os
from redis import Redis, RedisError
from pydantic import ValidationError
from src.config import setting
from src.models import User
class ProfileCache:
def __init__(self):
url = setting('CACHE_URL')
self.client = Redis.from_url(url, decode_responses=True, socket_connect_timeout=0.2,
socket_timeout=0.2, max_connections=16) if url else None
self.ttl = int(os.getenv('PROFILE_CACHE_TTL_SECONDS','60'))
self.hits = self.misses = self.errors = 0
def get(self, user_id):
if not self.client:
return None
try:
value = self.client.get('profile:v1:'+user_id)
if value:
user=User.model_validate_json(value)
if user.id != user_id:
raise ValueError('неверный ключ кэша')
self.hits += 1
return user
self.misses += 1
except (RedisError, ValidationError, ValueError):
self.errors += 1
logging.getLogger(__name__).warning('Кэш профиля недоступен; чтение из PostgreSQL')
return None
def put(self, user):
if self.client:
try:
self.client.set('profile:v1:'+user.id,user.model_dump_json(),ex=self.ttl)
except RedisError:
self.errors += 1
profile_cache = ProfileCache()
+12 -3
View File
@@ -15,10 +15,11 @@ from psycopg.errors import UniqueViolation, ForeignKeyViolation
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from src.config import setting
from src.cache import profile_cache
from src.models import User, Chat, Message
_CURSOR_FIELDS = {"scope", "resource_id", "created_at", "id"}
PASSWORDS = PasswordHasher()
PASSWORDS = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
DUMMY_HASH = PASSWORDS.hash('synthetic-dummy-password')
@@ -118,9 +119,15 @@ class PostgresStorage:
raise ConflictError('имя пользователя уже занято') from None
def get_user(self, user_id):
cached = profile_cache.get(user_id)
if cached:
return cached
with self.pool.connection() as conn:
row = conn.execute('SELECT id,username,display_name,created_at FROM users WHERE id=%s', (user_id,)).fetchone()
return User(**public_row(row)) if row else None
user = User(**public_row(row)) if row else None
if user:
profile_cache.put(user)
return user
def login(self, username, password):
with self.pool.connection() as conn:
@@ -225,7 +232,9 @@ class PostgresStorage:
def get_stats(self):
with self.pool.connection() as conn:
return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n'] for table in ('users','chats','messages')}
result = {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n'] for table in ('users','chats','messages')}
result['profile_cache'] = {'hits': profile_cache.hits, 'misses': profile_cache.misses, 'errors': profile_cache.errors, 'ttl_seconds': profile_cache.ttl}
return result
storage = PostgresStorage()
+18
View File
@@ -0,0 +1,18 @@
"""Проверить индекс истории на 10 000 синтетических сообщений."""
import json
from pathlib import Path
import sys
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
from client import Client, check
from support import sql
c=Client('http://localhost:8080')
a,b,_,chat=c.bootstrap()
sql("INSERT INTO messages (id,chat_id,sender_id,text,created_at) SELECT gen_random_uuid(),%s,%s,'Синтетическое ' || n, timestamptz '2026-01-01' + n * interval '1 microsecond' FROM generate_series(1,10000) n",[chat['id'],a['id']])
sql('ANALYZE messages')
marker=sql('SELECT created_at,id FROM messages WHERE chat_id=%s ORDER BY created_at,id OFFSET 8000 LIMIT 1',[chat['id']])[0]
query='SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT 51'
plan=sql('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) '+query,[chat['id'],marker['created_at'],marker['id']])[0]['QUERY PLAN'][0]
check('messages_history_idx' in json.dumps(plan),'индекс истории не используется')
Path('evidence/explain_10000.json').write_text(json.dumps({'rows':10000,'query':query,'plan':plan},ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
print('PASS: 10000 messages, indexed keyset pagination, execution ms:',plan['Execution Time'])
+21
View File
@@ -0,0 +1,21 @@
"""Административные запросы только к изолированному учебному стенду Compose."""
import json
import subprocess
def compose(*args):
result=subprocess.run(['docker','compose',*args],capture_output=True,text=True,encoding='utf-8',errors='replace',check=True)
return result.stdout.strip()
def sql(query, params=None):
source="""import json, psycopg
from psycopg.rows import dict_row
from src.config import setting
with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row) as conn:
cursor=conn.execute(%r, %r)
result=cursor.fetchall() if cursor.description else []
print(json.dumps(result,default=str))
""" % (query,params)
result=subprocess.run(['docker','compose','exec','-T','api','python','-'],input=source,capture_output=True,text=True,encoding='utf-8',check=True)
return json.loads(result.stdout)
+95
View File
@@ -0,0 +1,95 @@
"""Гонки, транзакции, короткий TTL и реальные отказы зависимостей."""
import concurrent.futures
import json
from pathlib import Path
import subprocess
import sys
import time
import uuid
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
from client import Client, check
from support import compose, sql
def wait_ready(client):
deadline=time.monotonic()+45
while time.monotonic()<deadline:
try:
client.call('GET','/health/ready',schema='Health')
return
except Exception:
time.sleep(0.3)
raise AssertionError('сервер не восстановился')
def main():
c=Client('http://localhost:8080')
alice,bob,eve,chat=c.bootstrap()
registration=dict(alice['registration'],username='race_'+uuid.uuid4().hex[:16])
def register(i):
# Оба результата допустимы для отдельного участника гонки.
import urllib.request, urllib.error
req=urllib.request.Request(c.base+'/api/v1/users',data=json.dumps(registration).encode(),headers={'Content-Type':'application/json'},method='POST')
try:
with urllib.request.urlopen(req,timeout=10) as response:
return response.status
except urllib.error.HTTPError as exc:
return exc.code
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
codes=list(pool.map(register,range(8)))
check(codes.count(201)==1 and codes.count(409)==7,'гонка регистрации: '+str(codes))
before=sql('SELECT count(*) AS n FROM chats')[0]['n']
c.call('POST','/api/v1/chats',400,actor=alice,body={'title':'Rollback','member_ids':[alice['id'],str(uuid.uuid4())]})
check(sql('SELECT count(*) AS n FROM chats')[0]['n']==before,'чат не откатился')
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
cached=compose('exec','-T','redis','redis-cli','GET','profile:v1:'+alice['id'])
check(alice['id'] in cached,'Redis не используется')
ttl=int(compose('exec','-T','redis','redis-cli','TTL','profile:v1:'+alice['id']))
check(0<ttl<=60,'нет TTL кэша')
started=time.monotonic()
try:
compose('stop','redis')
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
redis_fallback=time.monotonic()-started
c.call('GET','/api/v1/users/me',401)
finally:
compose('start','redis')
# Потеря кэша не теряет сессии и публичные профили.
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
try:
compose('stop','db')
c.call('GET','/health/live',schema='Health')
started=time.monotonic()
c.call('GET','/health/ready',503)
c.call('GET','/api/v1/users/me',503,actor=alice)
db_failure=time.monotonic()-started
finally:
compose('start','db')
wait_ready(c)
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
# В отдельном процессе срок жизни 2 секунды; основной стенд не меняется.
name='lyakhonov-lab2-expiry-'+uuid.uuid4().hex[:6]
try:
compose('run','-d','--name',name,'-p','127.0.0.1:8082:8080','-e','SESSION_TTL_SECONDS=2','api')
short=Client('http://localhost:8082')
wait_ready(short)
a,_,_,_=short.bootstrap()
short.call('GET','/api/v1/users/me',actor=a,schema='User')
time.sleep(2.2)
response=short.call('GET','/api/v1/users/me',401,actor=a)
check('Bearer' in response['headers'].get('WWW-Authenticate',''),'нет Bearer challenge')
finally:
subprocess.run(['docker','rm','-f',name],check=True,capture_output=True)
c.call('DELETE','/api/v1/auth/sessions/current',204,actor=alice)
c.call('GET','/api/v1/users/me',401,actor=alice)
result={'registration_codes':codes,'rollback':'PASS','profile_cache_ttl_seconds':ttl,
'redis_fallback':'PASS','redis_stop_and_request_seconds':round(redis_fallback,3),
'db_failure':'503, live=200, recovery with same session','two_db_error_requests_seconds':round(db_failure,3),
'expiry_seconds':2,'expiry':'401','logout':'401','passed':True}
Path('evidence/level4.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
print('PASS: registration race, chat rollback, Redis TTL/fallback/restart, DB outage/recovery, expiry, logout')
if __name__=='__main__':
main()