diff --git a/.gitattributes b/.gitattributes index 1a0e3e83..0b978c54 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,4 @@ *.py text eol=lf *.sql text eol=lf Dockerfile text eol=lf +*.cmd text eol=crlf diff --git a/Makefile b/Makefile index e5aeef8c..4c6df3c3 100644 --- a/Makefile +++ b/Makefile @@ -31,3 +31,10 @@ bootstrap: student-test: $(PYTHON) student-tests/test_lab2.py --base-url "$(BASE_URL)" + +.PHONY: level4-test explain +level4-test: + $(PYTHON) student-tests/test_level4.py + +explain: + $(PYTHON) student-tests/explain_level4.py diff --git a/compose.yaml b/compose.yaml index 0a528533..a3adea28 100644 --- a/compose.yaml +++ b/compose.yaml @@ -15,11 +15,24 @@ services: retries: 30 cpus: 1 mem_limit: 512m + redis: + image: redis:7.4.1-alpine + command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"] + volumes: [redis_data:/data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 3s + retries: 20 + cpus: 0.5 + mem_limit: 128m api: build: . image: lyakhonov-messenger:lab2 environment: DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL: redis://redis:6379/0 + PROFILE_CACHE_TTL_SECONDS: 60 SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} secrets: [database_url] ports: ["127.0.0.1:8080:8080"] @@ -35,4 +48,5 @@ secrets: database_url: file: ./secrets/database_url volumes: + redis_data: postgres_data: diff --git a/evidence/explain_10000.json b/evidence/explain_10000.json new file mode 100644 index 00000000..f8d8fa69 --- /dev/null +++ b/evidence/explain_10000.json @@ -0,0 +1,76 @@ +{ + "rows": 10000, + "query": "SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT 51", + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.29, + "Total Cost": 12.94, + "Plan Rows": 51, + "Plan Width": 87, + "Actual Startup Time": 0.017, + "Actual Total Time": 0.025, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.29, + "Total Cost": 536.25, + "Plan Rows": 2160, + "Plan Width": 87, + "Actual Startup Time": 0.016, + "Actual Total Time": 0.021, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '44cfe328-842b-43a5-900b-ff083cb13f7f'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.008001+00'::timestamp with time zone, '10e68164-4b26-406f-a3a4-f0395ca68f52'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 108, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.516, + "Triggers": [], + "Execution Time": 0.08 + } +} diff --git a/evidence/level4.json b/evidence/level4.json new file mode 100644 index 00000000..41d87fcb --- /dev/null +++ b/evidence/level4.json @@ -0,0 +1,22 @@ +{ + "registration_codes": [ + 201, + 409, + 409, + 409, + 409, + 409, + 409, + 409 + ], + "rollback": "PASS", + "profile_cache_ttl_seconds": 59, + "redis_fallback": "PASS", + "redis_stop_and_request_seconds": 8.617, + "db_failure": "503, live=200, recovery with same session", + "two_db_error_requests_seconds": 6.05, + "expiry_seconds": 2, + "expiry": "401", + "logout": "401", + "passed": true +} diff --git a/evidence/level4_smoke.txt b/evidence/level4_smoke.txt new file mode 100644 index 00000000..76a42482 --- /dev/null +++ b/evidence/level4_smoke.txt @@ -0,0 +1,2 @@ +python tests/smoke.py +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. diff --git a/infra/schema.md b/infra/schema.md index f1508363..eb4a6668 100644 --- a/infra/schema.md +++ b/infra/schema.md @@ -7,3 +7,6 @@ Индекс messages_history_idx (chat_id, created_at, id) обслуживает WHERE chat_id = ? AND (created_at,id) > (?,?) ORDER BY created_at,id LIMIT ?. Запрашиваются limit+1 строка, чтобы определить наличие следующей страницы. PostgreSQL timestamptz сохраняет микросекунды; UUID разрешает совпадения времени. Между страницами нет общего snapshot. Миграции выполняются под advisory lock в транзакции; schema_migrations предотвращает повторное применение. Пул ограничен десятью подключениями и 32 ожидающими; получение подключения — 3 секунды, statement_timeout — 3 секунды, lock_timeout — 2 секунды. + +## Кэш уровня 4 +Redis хранит только публичные неизменяемые профили пользователей, без хешей паролей и токенов. Ключ profile:v1:UUID, TTL 60 секунд, maxmemory 64 MiB, allkeys-lru. Пользовательские изменения и удаление не входят в API: инвалидация при обновлении не требуется; при будущем расширении она обязательна. Cache-aside: промах или ошибка возвращают чтение PostgreSQL; кэш заполняется после чтения подтверждённой строки. После рестарта кэш пуст и восстанавливается по запросам. Persistence кэша отключён намеренно; сессии находятся в PostgreSQL. Redis не критичен для readiness, PostgreSQL критичен. Кэш не участвует в проверке Bearer: каждая сессия проверяется в PostgreSQL, поэтому logout немедленно действует даже при недоступном Redis. diff --git a/requirements.lock.txt b/requirements.lock.txt index c6fe6332..ceec59ee 100644 --- a/requirements.lock.txt +++ b/requirements.lock.txt @@ -22,3 +22,4 @@ uvicorn==0.32.0 uvloop==0.23.0 watchfiles==1.3.0 websockets==17.2 +redis==5.2.0 diff --git a/requirements.txt b/requirements.txt index 9553a2dc..1b6204da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ pydantic==2.9.2 psycopg[binary]==3.2.3 psycopg-pool==3.2.4 argon2-cffi==23.1.0 +redis==5.2.0 diff --git a/scripts/restart.cmd b/scripts/restart.cmd index a79be87e..f3ca8a08 100644 --- a/scripts/restart.cmd +++ b/scripts/restart.cmd @@ -1,3 +1,3 @@ -@echo off -cd /d "%~dp0.." -docker compose restart db api +@echo off +cd /d "%~dp0.." +docker compose restart db redis api diff --git a/scripts/restart.sh b/scripts/restart.sh index b9b8009c..58adadc3 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,5 +1,5 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." -# Перезапускаем реальные процессы, сохраняя именованные тома. -docker compose restart db api +# Перезапускаем все процессы, не удаляя тома и не изменяя данные. +docker compose restart db redis api diff --git a/src/cache.py b/src/cache.py new file mode 100644 index 00000000..1bbf01ca --- /dev/null +++ b/src/cache.py @@ -0,0 +1,43 @@ +"""Необязательный кэш неизменяемого публичного профиля; ошибки обходятся через БД.""" +import logging +import os +from redis import Redis, RedisError +from pydantic import ValidationError +from src.config import setting +from src.models import User + + +class ProfileCache: + def __init__(self): + url = setting('CACHE_URL') + self.client = Redis.from_url(url, decode_responses=True, socket_connect_timeout=0.2, + socket_timeout=0.2, max_connections=16) if url else None + self.ttl = int(os.getenv('PROFILE_CACHE_TTL_SECONDS','60')) + self.hits = self.misses = self.errors = 0 + + def get(self, user_id): + if not self.client: + return None + try: + value = self.client.get('profile:v1:'+user_id) + if value: + user=User.model_validate_json(value) + if user.id != user_id: + raise ValueError('неверный ключ кэша') + self.hits += 1 + return user + self.misses += 1 + except (RedisError, ValidationError, ValueError): + self.errors += 1 + logging.getLogger(__name__).warning('Кэш профиля недоступен; чтение из PostgreSQL') + return None + + def put(self, user): + if self.client: + try: + self.client.set('profile:v1:'+user.id,user.model_dump_json(),ex=self.ttl) + except RedisError: + self.errors += 1 + + +profile_cache = ProfileCache() diff --git a/src/storage.py b/src/storage.py index ae62eeea..216e49d1 100644 --- a/src/storage.py +++ b/src/storage.py @@ -15,10 +15,11 @@ from psycopg.errors import UniqueViolation, ForeignKeyViolation from psycopg.rows import dict_row from psycopg_pool import ConnectionPool from src.config import setting +from src.cache import profile_cache from src.models import User, Chat, Message _CURSOR_FIELDS = {"scope", "resource_id", "created_at", "id"} -PASSWORDS = PasswordHasher() +PASSWORDS = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) DUMMY_HASH = PASSWORDS.hash('synthetic-dummy-password') @@ -118,9 +119,15 @@ class PostgresStorage: raise ConflictError('имя пользователя уже занято') from None def get_user(self, user_id): + cached = profile_cache.get(user_id) + if cached: + return cached with self.pool.connection() as conn: row = conn.execute('SELECT id,username,display_name,created_at FROM users WHERE id=%s', (user_id,)).fetchone() - return User(**public_row(row)) if row else None + user = User(**public_row(row)) if row else None + if user: + profile_cache.put(user) + return user def login(self, username, password): with self.pool.connection() as conn: @@ -225,7 +232,9 @@ class PostgresStorage: def get_stats(self): with self.pool.connection() as conn: - return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n'] for table in ('users','chats','messages')} + result = {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n'] for table in ('users','chats','messages')} + result['profile_cache'] = {'hits': profile_cache.hits, 'misses': profile_cache.misses, 'errors': profile_cache.errors, 'ttl_seconds': profile_cache.ttl} + return result storage = PostgresStorage() diff --git a/student-tests/explain_level4.py b/student-tests/explain_level4.py new file mode 100644 index 00000000..ddc94a9f --- /dev/null +++ b/student-tests/explain_level4.py @@ -0,0 +1,18 @@ +"""Проверить индекс истории на 10 000 синтетических сообщений.""" +import json +from pathlib import Path +import sys +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from client import Client, check +from support import sql + +c=Client('http://localhost:8080') +a,b,_,chat=c.bootstrap() +sql("INSERT INTO messages (id,chat_id,sender_id,text,created_at) SELECT gen_random_uuid(),%s,%s,'Синтетическое ' || n, timestamptz '2026-01-01' + n * interval '1 microsecond' FROM generate_series(1,10000) n",[chat['id'],a['id']]) +sql('ANALYZE messages') +marker=sql('SELECT created_at,id FROM messages WHERE chat_id=%s ORDER BY created_at,id OFFSET 8000 LIMIT 1',[chat['id']])[0] +query='SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT 51' +plan=sql('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) '+query,[chat['id'],marker['created_at'],marker['id']])[0]['QUERY PLAN'][0] +check('messages_history_idx' in json.dumps(plan),'индекс истории не используется') +Path('evidence/explain_10000.json').write_text(json.dumps({'rows':10000,'query':query,'plan':plan},ensure_ascii=False,indent=2)+'\n',encoding='utf-8') +print('PASS: 10000 messages, indexed keyset pagination, execution ms:',plan['Execution Time']) diff --git a/student-tests/support.py b/student-tests/support.py new file mode 100644 index 00000000..c323988b --- /dev/null +++ b/student-tests/support.py @@ -0,0 +1,21 @@ +"""Административные запросы только к изолированному учебному стенду Compose.""" +import json +import subprocess + + +def compose(*args): + result=subprocess.run(['docker','compose',*args],capture_output=True,text=True,encoding='utf-8',errors='replace',check=True) + return result.stdout.strip() + + +def sql(query, params=None): + source="""import json, psycopg +from psycopg.rows import dict_row +from src.config import setting +with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row) as conn: + cursor=conn.execute(%r, %r) + result=cursor.fetchall() if cursor.description else [] +print(json.dumps(result,default=str)) +""" % (query,params) + result=subprocess.run(['docker','compose','exec','-T','api','python','-'],input=source,capture_output=True,text=True,encoding='utf-8',check=True) + return json.loads(result.stdout) diff --git a/student-tests/test_level4.py b/student-tests/test_level4.py new file mode 100644 index 00000000..fcacb0f3 --- /dev/null +++ b/student-tests/test_level4.py @@ -0,0 +1,95 @@ +"""Гонки, транзакции, короткий TTL и реальные отказы зависимостей.""" +import concurrent.futures +import json +from pathlib import Path +import subprocess +import sys +import time +import uuid +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from client import Client, check +from support import compose, sql + + +def wait_ready(client): + deadline=time.monotonic()+45 + while time.monotonic()