Реализовал уровень 4: Redis-кэш, проверки отказов и индекс истории
This commit is contained in:
@@ -2,3 +2,4 @@
|
||||
*.py text eol=lf
|
||||
*.sql text eol=lf
|
||||
Dockerfile text eol=lf
|
||||
*.cmd text eol=crlf
|
||||
|
||||
@@ -31,3 +31,10 @@ bootstrap:
|
||||
|
||||
student-test:
|
||||
$(PYTHON) student-tests/test_lab2.py --base-url "$(BASE_URL)"
|
||||
|
||||
.PHONY: level4-test explain
|
||||
level4-test:
|
||||
$(PYTHON) student-tests/test_level4.py
|
||||
|
||||
explain:
|
||||
$(PYTHON) student-tests/explain_level4.py
|
||||
|
||||
@@ -15,11 +15,24 @@ services:
|
||||
retries: 30
|
||||
cpus: 1
|
||||
mem_limit: 512m
|
||||
redis:
|
||||
image: redis:7.4.1-alpine
|
||||
command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"]
|
||||
volumes: [redis_data:/data]
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
api:
|
||||
build: .
|
||||
image: lyakhonov-messenger:lab2
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
PROFILE_CACHE_TTL_SECONDS: 60
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
secrets: [database_url]
|
||||
ports: ["127.0.0.1:8080:8080"]
|
||||
@@ -35,4 +48,5 @@ secrets:
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
volumes:
|
||||
redis_data:
|
||||
postgres_data:
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"rows": 10000,
|
||||
"query": "SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT 51",
|
||||
"plan": {
|
||||
"Plan": {
|
||||
"Node Type": "Limit",
|
||||
"Parallel Aware": false,
|
||||
"Async Capable": false,
|
||||
"Startup Cost": 0.29,
|
||||
"Total Cost": 12.94,
|
||||
"Plan Rows": 51,
|
||||
"Plan Width": 87,
|
||||
"Actual Startup Time": 0.017,
|
||||
"Actual Total Time": 0.025,
|
||||
"Actual Rows": 51,
|
||||
"Actual Loops": 1,
|
||||
"Shared Hit Blocks": 5,
|
||||
"Shared Read Blocks": 0,
|
||||
"Shared Dirtied Blocks": 0,
|
||||
"Shared Written Blocks": 0,
|
||||
"Local Hit Blocks": 0,
|
||||
"Local Read Blocks": 0,
|
||||
"Local Dirtied Blocks": 0,
|
||||
"Local Written Blocks": 0,
|
||||
"Temp Read Blocks": 0,
|
||||
"Temp Written Blocks": 0,
|
||||
"Plans": [
|
||||
{
|
||||
"Node Type": "Index Scan",
|
||||
"Parent Relationship": "Outer",
|
||||
"Parallel Aware": false,
|
||||
"Async Capable": false,
|
||||
"Scan Direction": "Forward",
|
||||
"Index Name": "messages_history_idx",
|
||||
"Relation Name": "messages",
|
||||
"Alias": "messages",
|
||||
"Startup Cost": 0.29,
|
||||
"Total Cost": 536.25,
|
||||
"Plan Rows": 2160,
|
||||
"Plan Width": 87,
|
||||
"Actual Startup Time": 0.016,
|
||||
"Actual Total Time": 0.021,
|
||||
"Actual Rows": 51,
|
||||
"Actual Loops": 1,
|
||||
"Index Cond": "((chat_id = '44cfe328-842b-43a5-900b-ff083cb13f7f'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.008001+00'::timestamp with time zone, '10e68164-4b26-406f-a3a4-f0395ca68f52'::uuid)))",
|
||||
"Rows Removed by Index Recheck": 0,
|
||||
"Shared Hit Blocks": 5,
|
||||
"Shared Read Blocks": 0,
|
||||
"Shared Dirtied Blocks": 0,
|
||||
"Shared Written Blocks": 0,
|
||||
"Local Hit Blocks": 0,
|
||||
"Local Read Blocks": 0,
|
||||
"Local Dirtied Blocks": 0,
|
||||
"Local Written Blocks": 0,
|
||||
"Temp Read Blocks": 0,
|
||||
"Temp Written Blocks": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"Planning": {
|
||||
"Shared Hit Blocks": 108,
|
||||
"Shared Read Blocks": 0,
|
||||
"Shared Dirtied Blocks": 0,
|
||||
"Shared Written Blocks": 0,
|
||||
"Local Hit Blocks": 0,
|
||||
"Local Read Blocks": 0,
|
||||
"Local Dirtied Blocks": 0,
|
||||
"Local Written Blocks": 0,
|
||||
"Temp Read Blocks": 0,
|
||||
"Temp Written Blocks": 0
|
||||
},
|
||||
"Planning Time": 0.516,
|
||||
"Triggers": [],
|
||||
"Execution Time": 0.08
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"registration_codes": [
|
||||
201,
|
||||
409,
|
||||
409,
|
||||
409,
|
||||
409,
|
||||
409,
|
||||
409,
|
||||
409
|
||||
],
|
||||
"rollback": "PASS",
|
||||
"profile_cache_ttl_seconds": 59,
|
||||
"redis_fallback": "PASS",
|
||||
"redis_stop_and_request_seconds": 8.617,
|
||||
"db_failure": "503, live=200, recovery with same session",
|
||||
"two_db_error_requests_seconds": 6.05,
|
||||
"expiry_seconds": 2,
|
||||
"expiry": "401",
|
||||
"logout": "401",
|
||||
"passed": true
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python tests/smoke.py
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -7,3 +7,6 @@
|
||||
Индекс messages_history_idx (chat_id, created_at, id) обслуживает WHERE chat_id = ? AND (created_at,id) > (?,?) ORDER BY created_at,id LIMIT ?. Запрашиваются limit+1 строка, чтобы определить наличие следующей страницы. PostgreSQL timestamptz сохраняет микросекунды; UUID разрешает совпадения времени. Между страницами нет общего snapshot.
|
||||
|
||||
Миграции выполняются под advisory lock в транзакции; schema_migrations предотвращает повторное применение. Пул ограничен десятью подключениями и 32 ожидающими; получение подключения — 3 секунды, statement_timeout — 3 секунды, lock_timeout — 2 секунды.
|
||||
|
||||
## Кэш уровня 4
|
||||
Redis хранит только публичные неизменяемые профили пользователей, без хешей паролей и токенов. Ключ profile:v1:UUID, TTL 60 секунд, maxmemory 64 MiB, allkeys-lru. Пользовательские изменения и удаление не входят в API: инвалидация при обновлении не требуется; при будущем расширении она обязательна. Cache-aside: промах или ошибка возвращают чтение PostgreSQL; кэш заполняется после чтения подтверждённой строки. После рестарта кэш пуст и восстанавливается по запросам. Persistence кэша отключён намеренно; сессии находятся в PostgreSQL. Redis не критичен для readiness, PostgreSQL критичен. Кэш не участвует в проверке Bearer: каждая сессия проверяется в PostgreSQL, поэтому logout немедленно действует даже при недоступном Redis.
|
||||
|
||||
@@ -22,3 +22,4 @@ uvicorn==0.32.0
|
||||
uvloop==0.23.0
|
||||
watchfiles==1.3.0
|
||||
websockets==17.2
|
||||
redis==5.2.0
|
||||
|
||||
@@ -4,3 +4,4 @@ pydantic==2.9.2
|
||||
psycopg[binary]==3.2.3
|
||||
psycopg-pool==3.2.4
|
||||
argon2-cffi==23.1.0
|
||||
redis==5.2.0
|
||||
|
||||
+3
-3
@@ -1,3 +1,3 @@
|
||||
@echo off␍
|
||||
cd /d "%~dp0.."␍
|
||||
docker compose restart db api␍
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
docker compose restart db redis api
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Перезапускаем реальные процессы, сохраняя именованные тома.
|
||||
docker compose restart db api
|
||||
# Перезапускаем все процессы, не удаляя тома и не изменяя данные.
|
||||
docker compose restart db redis api
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Необязательный кэш неизменяемого публичного профиля; ошибки обходятся через БД."""
|
||||
import logging
|
||||
import os
|
||||
from redis import Redis, RedisError
|
||||
from pydantic import ValidationError
|
||||
from src.config import setting
|
||||
from src.models import User
|
||||
|
||||
|
||||
class ProfileCache:
|
||||
def __init__(self):
|
||||
url = setting('CACHE_URL')
|
||||
self.client = Redis.from_url(url, decode_responses=True, socket_connect_timeout=0.2,
|
||||
socket_timeout=0.2, max_connections=16) if url else None
|
||||
self.ttl = int(os.getenv('PROFILE_CACHE_TTL_SECONDS','60'))
|
||||
self.hits = self.misses = self.errors = 0
|
||||
|
||||
def get(self, user_id):
|
||||
if not self.client:
|
||||
return None
|
||||
try:
|
||||
value = self.client.get('profile:v1:'+user_id)
|
||||
if value:
|
||||
user=User.model_validate_json(value)
|
||||
if user.id != user_id:
|
||||
raise ValueError('неверный ключ кэша')
|
||||
self.hits += 1
|
||||
return user
|
||||
self.misses += 1
|
||||
except (RedisError, ValidationError, ValueError):
|
||||
self.errors += 1
|
||||
logging.getLogger(__name__).warning('Кэш профиля недоступен; чтение из PostgreSQL')
|
||||
return None
|
||||
|
||||
def put(self, user):
|
||||
if self.client:
|
||||
try:
|
||||
self.client.set('profile:v1:'+user.id,user.model_dump_json(),ex=self.ttl)
|
||||
except RedisError:
|
||||
self.errors += 1
|
||||
|
||||
|
||||
profile_cache = ProfileCache()
|
||||
+12
-3
@@ -15,10 +15,11 @@ from psycopg.errors import UniqueViolation, ForeignKeyViolation
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
from src.config import setting
|
||||
from src.cache import profile_cache
|
||||
from src.models import User, Chat, Message
|
||||
|
||||
_CURSOR_FIELDS = {"scope", "resource_id", "created_at", "id"}
|
||||
PASSWORDS = PasswordHasher()
|
||||
PASSWORDS = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
||||
DUMMY_HASH = PASSWORDS.hash('synthetic-dummy-password')
|
||||
|
||||
|
||||
@@ -118,9 +119,15 @@ class PostgresStorage:
|
||||
raise ConflictError('имя пользователя уже занято') from None
|
||||
|
||||
def get_user(self, user_id):
|
||||
cached = profile_cache.get(user_id)
|
||||
if cached:
|
||||
return cached
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('SELECT id,username,display_name,created_at FROM users WHERE id=%s', (user_id,)).fetchone()
|
||||
return User(**public_row(row)) if row else None
|
||||
user = User(**public_row(row)) if row else None
|
||||
if user:
|
||||
profile_cache.put(user)
|
||||
return user
|
||||
|
||||
def login(self, username, password):
|
||||
with self.pool.connection() as conn:
|
||||
@@ -225,7 +232,9 @@ class PostgresStorage:
|
||||
|
||||
def get_stats(self):
|
||||
with self.pool.connection() as conn:
|
||||
return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n'] for table in ('users','chats','messages')}
|
||||
result = {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n'] for table in ('users','chats','messages')}
|
||||
result['profile_cache'] = {'hits': profile_cache.hits, 'misses': profile_cache.misses, 'errors': profile_cache.errors, 'ttl_seconds': profile_cache.ttl}
|
||||
return result
|
||||
|
||||
|
||||
storage = PostgresStorage()
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Проверить индекс истории на 10 000 синтетических сообщений."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
|
||||
from client import Client, check
|
||||
from support import sql
|
||||
|
||||
c=Client('http://localhost:8080')
|
||||
a,b,_,chat=c.bootstrap()
|
||||
sql("INSERT INTO messages (id,chat_id,sender_id,text,created_at) SELECT gen_random_uuid(),%s,%s,'Синтетическое ' || n, timestamptz '2026-01-01' + n * interval '1 microsecond' FROM generate_series(1,10000) n",[chat['id'],a['id']])
|
||||
sql('ANALYZE messages')
|
||||
marker=sql('SELECT created_at,id FROM messages WHERE chat_id=%s ORDER BY created_at,id OFFSET 8000 LIMIT 1',[chat['id']])[0]
|
||||
query='SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT 51'
|
||||
plan=sql('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) '+query,[chat['id'],marker['created_at'],marker['id']])[0]['QUERY PLAN'][0]
|
||||
check('messages_history_idx' in json.dumps(plan),'индекс истории не используется')
|
||||
Path('evidence/explain_10000.json').write_text(json.dumps({'rows':10000,'query':query,'plan':plan},ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
||||
print('PASS: 10000 messages, indexed keyset pagination, execution ms:',plan['Execution Time'])
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Административные запросы только к изолированному учебному стенду Compose."""
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
|
||||
def compose(*args):
|
||||
result=subprocess.run(['docker','compose',*args],capture_output=True,text=True,encoding='utf-8',errors='replace',check=True)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def sql(query, params=None):
|
||||
source="""import json, psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from src.config import setting
|
||||
with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row) as conn:
|
||||
cursor=conn.execute(%r, %r)
|
||||
result=cursor.fetchall() if cursor.description else []
|
||||
print(json.dumps(result,default=str))
|
||||
""" % (query,params)
|
||||
result=subprocess.run(['docker','compose','exec','-T','api','python','-'],input=source,capture_output=True,text=True,encoding='utf-8',check=True)
|
||||
return json.loads(result.stdout)
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Гонки, транзакции, короткий TTL и реальные отказы зависимостей."""
|
||||
import concurrent.futures
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
|
||||
from client import Client, check
|
||||
from support import compose, sql
|
||||
|
||||
|
||||
def wait_ready(client):
|
||||
deadline=time.monotonic()+45
|
||||
while time.monotonic()<deadline:
|
||||
try:
|
||||
client.call('GET','/health/ready',schema='Health')
|
||||
return
|
||||
except Exception:
|
||||
time.sleep(0.3)
|
||||
raise AssertionError('сервер не восстановился')
|
||||
|
||||
|
||||
def main():
|
||||
c=Client('http://localhost:8080')
|
||||
alice,bob,eve,chat=c.bootstrap()
|
||||
registration=dict(alice['registration'],username='race_'+uuid.uuid4().hex[:16])
|
||||
def register(i):
|
||||
# Оба результата допустимы для отдельного участника гонки.
|
||||
import urllib.request, urllib.error
|
||||
req=urllib.request.Request(c.base+'/api/v1/users',data=json.dumps(registration).encode(),headers={'Content-Type':'application/json'},method='POST')
|
||||
try:
|
||||
with urllib.request.urlopen(req,timeout=10) as response:
|
||||
return response.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
codes=list(pool.map(register,range(8)))
|
||||
check(codes.count(201)==1 and codes.count(409)==7,'гонка регистрации: '+str(codes))
|
||||
before=sql('SELECT count(*) AS n FROM chats')[0]['n']
|
||||
c.call('POST','/api/v1/chats',400,actor=alice,body={'title':'Rollback','member_ids':[alice['id'],str(uuid.uuid4())]})
|
||||
check(sql('SELECT count(*) AS n FROM chats')[0]['n']==before,'чат не откатился')
|
||||
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
|
||||
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
|
||||
cached=compose('exec','-T','redis','redis-cli','GET','profile:v1:'+alice['id'])
|
||||
check(alice['id'] in cached,'Redis не используется')
|
||||
ttl=int(compose('exec','-T','redis','redis-cli','TTL','profile:v1:'+alice['id']))
|
||||
check(0<ttl<=60,'нет TTL кэша')
|
||||
started=time.monotonic()
|
||||
try:
|
||||
compose('stop','redis')
|
||||
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
|
||||
redis_fallback=time.monotonic()-started
|
||||
c.call('GET','/api/v1/users/me',401)
|
||||
finally:
|
||||
compose('start','redis')
|
||||
# Потеря кэша не теряет сессии и публичные профили.
|
||||
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
|
||||
try:
|
||||
compose('stop','db')
|
||||
c.call('GET','/health/live',schema='Health')
|
||||
started=time.monotonic()
|
||||
c.call('GET','/health/ready',503)
|
||||
c.call('GET','/api/v1/users/me',503,actor=alice)
|
||||
db_failure=time.monotonic()-started
|
||||
finally:
|
||||
compose('start','db')
|
||||
wait_ready(c)
|
||||
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
|
||||
# В отдельном процессе срок жизни 2 секунды; основной стенд не меняется.
|
||||
name='lyakhonov-lab2-expiry-'+uuid.uuid4().hex[:6]
|
||||
try:
|
||||
compose('run','-d','--name',name,'-p','127.0.0.1:8082:8080','-e','SESSION_TTL_SECONDS=2','api')
|
||||
short=Client('http://localhost:8082')
|
||||
wait_ready(short)
|
||||
a,_,_,_=short.bootstrap()
|
||||
short.call('GET','/api/v1/users/me',actor=a,schema='User')
|
||||
time.sleep(2.2)
|
||||
response=short.call('GET','/api/v1/users/me',401,actor=a)
|
||||
check('Bearer' in response['headers'].get('WWW-Authenticate',''),'нет Bearer challenge')
|
||||
finally:
|
||||
subprocess.run(['docker','rm','-f',name],check=True,capture_output=True)
|
||||
c.call('DELETE','/api/v1/auth/sessions/current',204,actor=alice)
|
||||
c.call('GET','/api/v1/users/me',401,actor=alice)
|
||||
result={'registration_codes':codes,'rollback':'PASS','profile_cache_ttl_seconds':ttl,
|
||||
'redis_fallback':'PASS','redis_stop_and_request_seconds':round(redis_fallback,3),
|
||||
'db_failure':'503, live=200, recovery with same session','two_db_error_requests_seconds':round(db_failure,3),
|
||||
'expiry_seconds':2,'expiry':'401','logout':'401','passed':True}
|
||||
Path('evidence/level4.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
||||
print('PASS: registration race, chat rollback, Redis TTL/fallback/restart, DB outage/recovery, expiry, logout')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user