forked from mtusi2026/mtusi2026lab2
lab2
This commit is contained in:
+4
-1
@@ -7,5 +7,8 @@ SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
SESSION_STORE=redis
|
||||
# Local compose uses DATABASE_URL/CACHE_URL directly; file paths optional in production.
|
||||
DATABASE_URL=postgresql://messenger:messenger_dev_only@localhost:5432/messenger
|
||||
CACHE_URL=redis://localhost:6379/0
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
FROM python:3.12.11-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY app.py auth.py db.py sessions.py ./
|
||||
COPY migrations ./migrations
|
||||
COPY contracts/openapi.json ./contracts/openapi.json
|
||||
RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser \
|
||||
&& chown -R 10001:10001 /app
|
||||
USER 10001:10001
|
||||
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" || exit 1
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
CMD ["python", "app.py"]
|
||||
@@ -4,7 +4,20 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence
|
||||
.PHONY: help check validate test load test-persistence student-test test-grade4-infra test-grade4-expiry track2
|
||||
|
||||
track2:
|
||||
$(PYTHON) scripts/track2_run.py
|
||||
|
||||
student-test:
|
||||
$(PYTHON) -m unittest discover -s student-tests -p 'test_*.py'
|
||||
|
||||
test-grade4-infra:
|
||||
RUN_INFRA_TEST=1 $(PYTHON) -m unittest discover -s student-tests -p 'test_dependencies.py'
|
||||
|
||||
test-grade4-expiry:
|
||||
@echo "Start stack with: docker compose -f compose.yaml -f compose.ttl.yaml up -d --build"
|
||||
@echo "Then: RUN_EXPIRY_TEST=1 $(PYTHON) -m unittest student-tests.test_lab2.Lab2Grade4Tests.test_session_expiry"
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
|
||||
@@ -1,37 +1,88 @@
|
||||
# Отчёт — лабораторная 2
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Журавлев Алексей, БПИ2301
|
||||
- Целевая оценка: **5** — уровень 4 + **трек 2** (план запроса и восстановление)
|
||||
- Commit/PR: TODO
|
||||
- Источники: lab1, skeleton lab2, PostgreSQL EXPLAIN/backup docs; помощь ИИ при реализации и экспериментах
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Prerequisites: Docker Desktop, Python 3.9+.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
docker compose up -d --build
|
||||
py -3 tests/smoke.py --base-url http://localhost:8080
|
||||
py -3 -m unittest discover -s student-tests -p 'test_*.py'
|
||||
RUN_INFRA_TEST=1 py -3 -m unittest student-tests.test_dependencies
|
||||
chmod +x scripts/restart.sh
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Windows persistence: `py -3 tests/resilience.py --mode persistence --action-script scripts/restart.bat`
|
||||
|
||||
## Доказательства критериев
|
||||
Трек 5 (трек 2): `py -3 scripts/track2_run.py` → `RUN_TRACK2_VERIFY=1 py -3 -m unittest student-tests.test_track2`
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Expiry: `docker compose -f compose.yaml -f compose.ttl.yaml up -d --build` → `RUN_EXPIRY_TEST=1 py -3 -m unittest student-tests.test_lab2.Lab2Grade4Tests.test_session_expiry` → вернуть обычный compose.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Образы: `python:3.12.11-slim-bookworm`, `postgres:16.8-alpine`, `redis:7.4.2-alpine`.
|
||||
|
||||
## Эксперименты и отказы
|
||||
## Архитектура
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
- **PostgreSQL** — users/chats/messages; миграции `migrations/`; индекс `messages_chat_history_idx (chat_id, created_at, id)`.
|
||||
- **Redis AOF** — сессии `sess:<token>`, TTL `SESSION_TTL_SECONDS`.
|
||||
- **API** — Bearer/Basic по контракту lab2; keyset-пагинация истории.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
## Доказательства уровня 3–4
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
| Критерий | Команда | Результат |
|
||||
| --- | --- | --- |
|
||||
| Smoke | `py -3 tests/smoke.py` | PASS |
|
||||
| Persistence | `resilience.py` + restart hook | PASS |
|
||||
| Student auth/race/rollback | `make student-test` | PASS |
|
||||
| Redis/Postgres down | `RUN_INFRA_TEST=1` … `test_dependencies` | PASS |
|
||||
| Session expiry | `compose.ttl.yaml` + `RUN_EXPIRY_TEST=1` | PASS |
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
## Трек 2 (оценка 5)
|
||||
|
||||
### Гипотеза
|
||||
|
||||
1. Keyset `LIMIT 50` на hot-чате (~70k сообщений) использует `messages_chat_history_idx` и быстрее, чем `OFFSET 50000`.
|
||||
2. Дополнительный индекс только на `(chat_id)` не улучшает keyset, но снижает скорость INSERT (~11% на локальном прогоне).
|
||||
3. Логический `pg_dump` → новый volume (`compose.track2-restore.yaml`) восстанавливает те же counts и контрольные сообщения; RPO ограничен моментом dump (без PITR).
|
||||
|
||||
### Методика
|
||||
|
||||
- Seed: `scripts/track2_seed.py` — 100 000 сообщений, 70% в `track2_hot`, остальное в medium/small чатах.
|
||||
- Benchmark: `scripts/track2_benchmark.py` — `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` для keyset vs OFFSET vs лишний индекс; 500 INSERT до/после лишнего индекса.
|
||||
- Backup/restore: `scripts/track2_backup_restore.py` — `pg_dump -Fc` → `postgres_restore` на порту 5433 → сверка counts + 5 последних сообщений.
|
||||
|
||||
Стенд: Docker Desktop, один host, без CPU/RAM limits (учебный ноутбук).
|
||||
|
||||
### Результаты (2026-10-23)
|
||||
|
||||
| Сценарий | Время / метрика | Индекс |
|
||||
| --- | --- | --- |
|
||||
| Keyset LIMIT 50 (hot) | **0.35 ms** | `messages_chat_history_idx` (Index Only Scan) |
|
||||
| OFFSET 50000 LIMIT 50 | **7.04 ms** | тот же индекс, но дороже из‑за offset |
|
||||
| Keyset + лишний `(chat_id)` | **9.80 ms** | planner выбрал худший план |
|
||||
| INSERT 500 msg/s | 20167 без лишнего индекса vs 18200 с лишним | −~11% write throughput |
|
||||
|
||||
Restore: 111 012 сообщений, dump 2.9 MiB, **7.1 s** end-to-end, counts/sample match — см. `evidence/track2/restore_report.json`.
|
||||
|
||||
### Ограничения
|
||||
|
||||
- Один PostgreSQL instance; нет WAL archiving / PITR — при аварии между dump и restore возможна потеря записей.
|
||||
- Benchmark на синтетических однобуквенных сообщениях; абсолютные ms зависят от железа.
|
||||
- Трек 1 (кэш истории) не выбран: Redis используется только для сессий.
|
||||
|
||||
## Отказы
|
||||
|
||||
| Воздействие | Наблюдение |
|
||||
| --- | --- |
|
||||
| restart api+postgres+redis (volumes) | данные + Bearer-сессия сохраняются |
|
||||
| stop redis | ready 503, `/users/me` 503 |
|
||||
| stop postgres | ready 503 |
|
||||
|
||||
## Следующие шаги (lab3+)
|
||||
|
||||
Горизонтальное масштабирование API, общие сессии Redis, idempotency сообщений.
|
||||
|
||||
@@ -0,0 +1,488 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from auth import (
|
||||
create_session,
|
||||
hash_password,
|
||||
now_iso,
|
||||
parse_basic_auth,
|
||||
parse_bearer_token,
|
||||
validate_password,
|
||||
verify_password,
|
||||
)
|
||||
from db import ApiError, Store
|
||||
from sessions import SessionStore
|
||||
|
||||
MAX_BODY_BYTES = 1024 * 1024
|
||||
USERNAME_PATTERN = re.compile(r'^[a-z0-9_]{3,32}$')
|
||||
CURSOR_TIMESTAMP_PATTERN = re.compile(
|
||||
r'^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?Z$'
|
||||
)
|
||||
LOG = logging.getLogger('messenger')
|
||||
OPENAPI_PATH = Path(__file__).resolve().parent / 'contracts' / 'openapi.json'
|
||||
SWAGGER_HTML = b'''<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>MTUSI Messenger API</title>
|
||||
<link rel="stylesheet" href="https://unpkg.com/swagger-ui-dist@5.17.14/swagger-ui.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="swagger-ui"></div>
|
||||
<script src="https://unpkg.com/swagger-ui-dist@5.17.14/swagger-ui-bundle.js"></script>
|
||||
<script>
|
||||
window.onload = () => SwaggerUIBundle({
|
||||
url: "/openapi.json",
|
||||
dom_id: "#swagger-ui",
|
||||
deepLinking: true,
|
||||
presets: [SwaggerUIBundle.presets.apis],
|
||||
layout: "BaseLayout"
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
|
||||
|
||||
def cursor_encode(resource, item):
|
||||
value = json.dumps(
|
||||
{'resource': resource, 'created_at': item['created_at'], 'id': item['id']},
|
||||
separators=(',', ':'),
|
||||
).encode('utf-8')
|
||||
return base64.urlsafe_b64encode(value).rstrip(b'=').decode('ascii')
|
||||
|
||||
|
||||
def cursor_decode(cursor, resource):
|
||||
try:
|
||||
padding = '=' * (-len(cursor) % 4)
|
||||
decoded = base64.b64decode(cursor + padding, altchars=b'-_', validate=True)
|
||||
value = json.loads(decoded.decode('utf-8'))
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid cursor') from None
|
||||
if (not isinstance(value, dict) or set(value) != {'resource', 'created_at', 'id'}
|
||||
or value.get('resource') != resource
|
||||
or not isinstance(value.get('created_at'), str)
|
||||
or not CURSOR_TIMESTAMP_PATTERN.fullmatch(value['created_at'])
|
||||
or not isinstance(value.get('id'), str)):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid cursor')
|
||||
try:
|
||||
created_at = datetime.fromisoformat(value['created_at'][:-1] + '+00:00')
|
||||
item_id = str(uuid.UUID(value['id']))
|
||||
except (ValueError, AttributeError):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid cursor') from None
|
||||
if item_id != value['id']:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid cursor')
|
||||
return created_at, item_id
|
||||
|
||||
|
||||
def required_text(value, name, maximum):
|
||||
if not isinstance(value, str) or not value.strip() or len(value) > maximum:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid ' + name)
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeEncodeError:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid ' + name) from None
|
||||
return value
|
||||
|
||||
|
||||
def parse_uuid(value):
|
||||
if not isinstance(value, str):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid UUID')
|
||||
try:
|
||||
parsed = uuid.UUID(value)
|
||||
except (ValueError, AttributeError):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid UUID') from None
|
||||
if str(parsed) != value.lower():
|
||||
raise ApiError(400, 'invalid_request', 'Invalid UUID')
|
||||
return str(parsed)
|
||||
|
||||
|
||||
class MessengerHandler(BaseHTTPRequestHandler):
|
||||
server_version = 'MTUSI-Messenger'
|
||||
protocol_version = 'HTTP/1.1'
|
||||
|
||||
def log_message(self, _format, *args):
|
||||
LOG.info('request method=%s status=%s request_id=%s', self.command, self._status, self._request_id)
|
||||
|
||||
def _send_bytes(self, status, request_id, body, content_type, extra_headers=None):
|
||||
self._status = status
|
||||
self.send_response(status)
|
||||
self.send_header('X-Request-Id', request_id)
|
||||
self.send_header('Content-Length', str(len(body)))
|
||||
if status != 204:
|
||||
self.send_header('Content-Type', content_type)
|
||||
for name, value in (extra_headers or {}).items():
|
||||
self.send_header(name, value)
|
||||
self.end_headers()
|
||||
if body:
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send(self, status, request_id, payload=None, extra_headers=None):
|
||||
body = b'' if status == 204 else json.dumps(
|
||||
payload, ensure_ascii=False, separators=(',', ':')
|
||||
).encode('utf-8')
|
||||
self._send_bytes(
|
||||
status, request_id, body, 'application/json; charset=utf-8', extra_headers
|
||||
)
|
||||
|
||||
def _error(self, request_id, error, extra_headers=None):
|
||||
self._send(error.status, request_id, {
|
||||
'error': {
|
||||
'code': error.code,
|
||||
'message': error.message,
|
||||
'request_id': request_id,
|
||||
},
|
||||
}, extra_headers=extra_headers)
|
||||
|
||||
def _read_json(self):
|
||||
if self.headers.get_content_type() != 'application/json':
|
||||
raise ApiError(415, 'unsupported_media_type', 'Expected application/json')
|
||||
raw_length = self.headers.get('Content-Length')
|
||||
try:
|
||||
length = int(raw_length)
|
||||
except (TypeError, ValueError):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid Content-Length') from None
|
||||
if length < 0:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid Content-Length')
|
||||
if length > MAX_BODY_BYTES:
|
||||
raise ApiError(413, 'payload_too_large', 'Request body is too large')
|
||||
try:
|
||||
value = json.loads(self.rfile.read(length).decode('utf-8'))
|
||||
except (ValueError, UnicodeError):
|
||||
raise ApiError(400, 'invalid_request', 'Malformed JSON') from None
|
||||
if not isinstance(value, dict):
|
||||
raise ApiError(400, 'invalid_request', 'Expected a JSON object')
|
||||
return value
|
||||
|
||||
def _dependencies_ok(self):
|
||||
try:
|
||||
self.server.store.ping()
|
||||
self.server.sessions.ping()
|
||||
return True
|
||||
except Exception:
|
||||
LOG.exception('dependency check failed')
|
||||
return False
|
||||
|
||||
def _unauthorized_bearer(self, request_id):
|
||||
self._error(
|
||||
request_id,
|
||||
ApiError(401, 'unauthorized', 'Authentication required'),
|
||||
extra_headers={'WWW-Authenticate': 'Bearer realm="messenger"'},
|
||||
)
|
||||
|
||||
def _unauthorized_basic(self, request_id):
|
||||
self._error(
|
||||
request_id,
|
||||
ApiError(401, 'unauthorized', 'Authentication required'),
|
||||
extra_headers={'WWW-Authenticate': 'Basic realm="messenger", charset="UTF-8"'},
|
||||
)
|
||||
|
||||
def _authenticated_user(self, request_id):
|
||||
auth = self.headers.get('Authorization', '')
|
||||
if auth.startswith('Basic '):
|
||||
self._unauthorized_bearer(request_id)
|
||||
return None
|
||||
token = parse_bearer_token(auth)
|
||||
if not token:
|
||||
self._unauthorized_bearer(request_id)
|
||||
return None
|
||||
try:
|
||||
user_id = self.server.sessions.get_user_id(token)
|
||||
except Exception:
|
||||
LOG.exception('session lookup failed')
|
||||
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
|
||||
return None
|
||||
if not user_id:
|
||||
self._unauthorized_bearer(request_id)
|
||||
return None
|
||||
try:
|
||||
user = self.server.store.get_user(user_id)
|
||||
except Exception:
|
||||
LOG.exception('user lookup failed')
|
||||
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
|
||||
return None
|
||||
if user is None:
|
||||
self._unauthorized_bearer(request_id)
|
||||
return None
|
||||
user['_token'] = token
|
||||
return user
|
||||
|
||||
@staticmethod
|
||||
def _query(query):
|
||||
try:
|
||||
values = parse_qs(query, keep_blank_values=True, strict_parsing=True)
|
||||
except ValueError:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid query parameters') from None
|
||||
if set(values) - {'limit', 'cursor'} or any(len(items) != 1 for items in values.values()):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid query parameters')
|
||||
try:
|
||||
limit = int(values.get('limit', ['50'])[0])
|
||||
except ValueError:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid limit') from None
|
||||
if not 1 <= limit <= 100:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid limit')
|
||||
cursor = values.get('cursor', [''])[0]
|
||||
if ('cursor' in values and not cursor) or len(cursor) > 2048:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid cursor')
|
||||
return limit, cursor
|
||||
|
||||
def _create_session(self, request_id):
|
||||
credentials = parse_basic_auth(self.headers.get('Authorization'))
|
||||
if not credentials:
|
||||
self._unauthorized_basic(request_id)
|
||||
return
|
||||
username, password = credentials
|
||||
if not isinstance(username, str) or not USERNAME_PATTERN.fullmatch(username):
|
||||
self._unauthorized_basic(request_id)
|
||||
return
|
||||
if not validate_password(password):
|
||||
self._unauthorized_basic(request_id)
|
||||
return
|
||||
try:
|
||||
row = self.server.store.get_user_with_password(username)
|
||||
except Exception:
|
||||
LOG.exception('login lookup failed')
|
||||
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
|
||||
return
|
||||
if row is None or not verify_password(row['password_hash'], password):
|
||||
self._unauthorized_basic(request_id)
|
||||
return
|
||||
user = self.server.store.get_user(str(row['id']))
|
||||
try:
|
||||
token, expires_at = create_session(self.server.sessions, user['id'])
|
||||
except Exception:
|
||||
LOG.exception('session create failed')
|
||||
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
|
||||
return
|
||||
self._send(201, request_id, {
|
||||
'token': token,
|
||||
'token_type': 'Bearer',
|
||||
'expires_at': expires_at,
|
||||
'user': user,
|
||||
})
|
||||
|
||||
def _logout(self, request_id, user):
|
||||
try:
|
||||
self.server.sessions.revoke(user['_token'])
|
||||
except Exception:
|
||||
LOG.exception('session revoke failed')
|
||||
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
|
||||
return
|
||||
self._send_bytes(204, request_id, b'', 'application/json; charset=utf-8')
|
||||
|
||||
def _register(self, request_id, body):
|
||||
if set(body) != {'username', 'display_name', 'password'}:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid user fields')
|
||||
username = body['username']
|
||||
if not isinstance(username, str) or not USERNAME_PATTERN.fullmatch(username):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid username')
|
||||
display_name = required_text(body['display_name'], 'display_name', 100)
|
||||
password = body['password']
|
||||
if not validate_password(password):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid password')
|
||||
created_at = now_iso()
|
||||
password_hash = hash_password(password)
|
||||
user = self.server.store.register_user(username, display_name, password_hash, created_at)
|
||||
self._send(201, request_id, user)
|
||||
|
||||
def _dispatch(self):
|
||||
request_id = str(uuid.uuid4())
|
||||
self._request_id = request_id
|
||||
self._status = 500
|
||||
try:
|
||||
if self.server.stopping:
|
||||
raise ApiError(503, 'unavailable', 'Server is shutting down')
|
||||
route = urlsplit(self.path)
|
||||
path = route.path
|
||||
if self.command == 'GET' and path in ('/docs', '/docs/'):
|
||||
self._send_bytes(200, request_id, SWAGGER_HTML, 'text/html; charset=utf-8')
|
||||
return
|
||||
if self.command == 'GET' and path == '/openapi.json':
|
||||
self._send_bytes(
|
||||
200, request_id, self.server.openapi_json, 'application/json; charset=utf-8'
|
||||
)
|
||||
return
|
||||
if path == '/health/live' and self.command == 'GET':
|
||||
self._send(200, request_id, {'status': 'ok'})
|
||||
return
|
||||
if path == '/health/ready' and self.command == 'GET':
|
||||
if self._dependencies_ok():
|
||||
self._send(200, request_id, {'status': 'ok'})
|
||||
else:
|
||||
self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable'))
|
||||
return
|
||||
if path == '/api/v1/users' and self.command == 'POST':
|
||||
self._register(request_id, self._read_json())
|
||||
return
|
||||
if path == '/api/v1/auth/sessions' and self.command == 'POST':
|
||||
self._create_session(request_id)
|
||||
return
|
||||
if path.startswith('/api/v1/'):
|
||||
user = self._authenticated_user(request_id)
|
||||
if user is None:
|
||||
return
|
||||
user_id = user['id']
|
||||
if path == '/api/v1/auth/sessions/current' and self.command == 'DELETE':
|
||||
self._logout(request_id, user)
|
||||
return
|
||||
if path == '/api/v1/users/me' and self.command == 'GET':
|
||||
self._send(200, request_id, {
|
||||
k: user[k] for k in ('id', 'username', 'display_name', 'created_at')
|
||||
})
|
||||
return
|
||||
if path == '/api/v1/chats' and self.command == 'POST':
|
||||
body = self._read_json()
|
||||
if set(body) != {'title', 'member_ids'}:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid chat fields')
|
||||
title = required_text(body['title'], 'title', 100)
|
||||
member_ids = body['member_ids']
|
||||
if (not isinstance(member_ids, list) or not member_ids or len(member_ids) > 100
|
||||
or any(not isinstance(value, str) for value in member_ids)):
|
||||
raise ApiError(400, 'invalid_request', 'Invalid chat members')
|
||||
normalized = [parse_uuid(value) for value in member_ids]
|
||||
if len(set(normalized)) != len(normalized) or user_id not in normalized:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid chat members')
|
||||
chat = self.server.store.create_chat(
|
||||
user_id, title, normalized, now_iso()
|
||||
)
|
||||
self._send(201, request_id, chat)
|
||||
return
|
||||
if path == '/api/v1/chats' and self.command == 'GET':
|
||||
limit, cursor = self._query(route.query)
|
||||
key = cursor_decode(cursor, 'chats') if cursor else None
|
||||
items, next_key = self.server.store.list_chats(user_id, limit, key)
|
||||
next_cursor = cursor_encode('chats', items[-1]) if next_key else None
|
||||
self._send(200, request_id, {'items': items, 'next_cursor': next_cursor})
|
||||
return
|
||||
match = re.fullmatch(r'/api/v1/chats/([^/]+)', path)
|
||||
if match and self.command == 'GET':
|
||||
chat_id = parse_uuid(match.group(1))
|
||||
self._send(200, request_id, self.server.store.get_chat(user_id, chat_id))
|
||||
return
|
||||
match = re.fullmatch(r'/api/v1/chats/([^/]+)/messages', path)
|
||||
if match:
|
||||
chat_id = parse_uuid(match.group(1))
|
||||
resource = 'messages:' + chat_id
|
||||
if self.command == 'POST':
|
||||
body = self._read_json()
|
||||
if not isinstance(body, dict) or set(body) != {'text'}:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid message fields')
|
||||
text = body['text']
|
||||
if not isinstance(text, str) or len(text) > 4000 or not text.strip():
|
||||
raise ApiError(400, 'invalid_request', 'Message text must not be empty')
|
||||
try:
|
||||
text.encode('utf-8')
|
||||
except UnicodeEncodeError:
|
||||
raise ApiError(400, 'invalid_request', 'Invalid message text') from None
|
||||
message = self.server.store.create_message(
|
||||
user_id, chat_id, text, now_iso()
|
||||
)
|
||||
self._send(201, request_id, message)
|
||||
return
|
||||
if self.command == 'GET':
|
||||
limit, cursor = self._query(route.query)
|
||||
key = cursor_decode(cursor, resource) if cursor else None
|
||||
items, next_key = self.server.store.list_messages(user_id, chat_id, limit, key)
|
||||
next_cursor = cursor_encode(resource, items[-1]) if next_key else None
|
||||
self._send(200, request_id, {'items': items, 'next_cursor': next_cursor})
|
||||
return
|
||||
raise ApiError(404, 'not_found', 'Resource not found')
|
||||
except ApiError as error:
|
||||
self._error(request_id, error)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
LOG.info('request disconnected request_id=%s', request_id)
|
||||
except Exception:
|
||||
LOG.exception('request failed request_id=%s', request_id)
|
||||
self._error(request_id, ApiError(500, 'internal_error', 'Internal server error'))
|
||||
|
||||
def do_GET(self):
|
||||
self._dispatch()
|
||||
|
||||
def do_POST(self):
|
||||
self._dispatch()
|
||||
|
||||
def do_PUT(self):
|
||||
self._dispatch()
|
||||
|
||||
def do_PATCH(self):
|
||||
self._dispatch()
|
||||
|
||||
def do_DELETE(self):
|
||||
self._dispatch()
|
||||
|
||||
def do_OPTIONS(self):
|
||||
self._dispatch()
|
||||
|
||||
|
||||
class MessengerServer(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
block_on_close = False
|
||||
|
||||
def __init__(self, address, store, sessions):
|
||||
super().__init__(address, MessengerHandler)
|
||||
self.store = store
|
||||
self.sessions = sessions
|
||||
self.stopping = False
|
||||
try:
|
||||
spec = json.loads(OPENAPI_PATH.read_text(encoding='utf-8'))
|
||||
spec['servers'] = [{'url': '/', 'description': 'Текущий адрес API'}]
|
||||
self.openapi_json = json.dumps(
|
||||
spec, ensure_ascii=False, separators=(',', ':')
|
||||
).encode('utf-8')
|
||||
except (OSError, ValueError) as error:
|
||||
self.server_close()
|
||||
raise RuntimeError('Unable to load the OpenAPI contract') from error
|
||||
|
||||
|
||||
def main():
|
||||
host = os.environ.get('APP_HOST', '0.0.0.0')
|
||||
try:
|
||||
port = int(os.environ.get('APP_PORT', '8080'))
|
||||
except ValueError:
|
||||
raise SystemExit('APP_PORT must be an integer')
|
||||
if not 1 <= port <= 65535:
|
||||
raise SystemExit('APP_PORT must be between 1 and 65535')
|
||||
if os.environ.get('SESSION_STORE', 'redis').lower() != 'redis':
|
||||
raise SystemExit('SESSION_STORE must be redis for this implementation')
|
||||
|
||||
logging.basicConfig(
|
||||
level=os.environ.get('LOG_LEVEL', 'INFO').upper(),
|
||||
format='%(asctime)s %(levelname)s %(name)s %(message)s',
|
||||
stream=sys.stdout,
|
||||
)
|
||||
store = Store.connect()
|
||||
sessions = SessionStore.from_env()
|
||||
server = MessengerServer((host, port), store, sessions)
|
||||
|
||||
def stop(_signum, _frame):
|
||||
server.stopping = True
|
||||
threading.Thread(target=server.shutdown, name='http-shutdown', daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
LOG.info('server started host=%s port=%d', host, port)
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.2)
|
||||
finally:
|
||||
server.server_close()
|
||||
store.close()
|
||||
LOG.info('server stopped')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,77 @@
|
||||
import base64
|
||||
import binascii
|
||||
import os
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerifyMismatchError
|
||||
|
||||
from sessions import SessionStore
|
||||
|
||||
PASSWORD_MIN = 12
|
||||
PASSWORD_MAX = 128
|
||||
_hasher = PasswordHasher()
|
||||
|
||||
|
||||
def now_iso():
|
||||
return datetime.now(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def validate_password(password):
|
||||
if not isinstance(password, str):
|
||||
return False
|
||||
length = len(password)
|
||||
return PASSWORD_MIN <= length <= PASSWORD_MAX
|
||||
|
||||
|
||||
def hash_password(password):
|
||||
return _hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash, password):
|
||||
try:
|
||||
return _hasher.verify(password_hash, password)
|
||||
except VerifyMismatchError:
|
||||
return False
|
||||
|
||||
|
||||
def parse_basic_auth(header_value):
|
||||
if not header_value or not header_value.startswith('Basic '):
|
||||
return None
|
||||
encoded = header_value[6:].strip()
|
||||
try:
|
||||
decoded = base64.b64decode(encoded, validate=True).decode('utf-8')
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
return None
|
||||
if ':' not in decoded:
|
||||
return None
|
||||
username, password = decoded.split(':', 1)
|
||||
if not username:
|
||||
return None
|
||||
return username, password
|
||||
|
||||
|
||||
def parse_bearer_token(header_value):
|
||||
if not header_value or not header_value.startswith('Bearer '):
|
||||
return None
|
||||
token = header_value[7:].strip()
|
||||
return token or None
|
||||
|
||||
|
||||
def session_ttl_seconds():
|
||||
try:
|
||||
ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600'))
|
||||
except ValueError:
|
||||
ttl = 3600
|
||||
return max(1, ttl)
|
||||
|
||||
|
||||
def create_session(store: SessionStore, user_id):
|
||||
token = secrets.token_urlsafe(32)
|
||||
ttl = session_ttl_seconds()
|
||||
expires_at = (
|
||||
datetime.now(timezone.utc) + timedelta(seconds=ttl)
|
||||
).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
store.create(token, user_id, ttl)
|
||||
return token, expires_at
|
||||
@@ -0,0 +1,19 @@
|
||||
services:
|
||||
postgres_restore:
|
||||
image: postgres:16.8-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD: messenger_dev_only
|
||||
POSTGRES_DB: postgres
|
||||
ports:
|
||||
- '5433:5432'
|
||||
volumes:
|
||||
- postgres_restore_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'pg_isready -U messenger']
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
volumes:
|
||||
postgres_restore_data:
|
||||
@@ -0,0 +1,5 @@
|
||||
# Overlay for session expiry checks: docker compose -f compose.yaml -f compose.ttl.yaml up -d --build
|
||||
services:
|
||||
api:
|
||||
environment:
|
||||
SESSION_TTL_SECONDS: 2
|
||||
@@ -0,0 +1,48 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.8-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD: messenger_dev_only
|
||||
POSTGRES_DB: messenger
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'pg_isready -U messenger -d messenger']
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
ports:
|
||||
- '5432:5432'
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
command: ['redis-server', '--appendonly', 'yes', '--appendfsync', 'everysec']
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ['CMD', 'redis-cli', 'ping']
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
api:
|
||||
build: .
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
SESSION_STORE: redis
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DATABASE_URL: postgresql://messenger:messenger_dev_only@postgres:5432/messenger
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
ports:
|
||||
- '${APP_PORT:-8080}:8080'
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
@@ -0,0 +1,256 @@
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from psycopg import errors
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
MIGRATIONS_DIR = Path(__file__).resolve().parent / 'migrations'
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, code, message):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
def read_database_url():
|
||||
path = os.environ.get('DATABASE_URL_FILE')
|
||||
if path and os.path.isfile(path):
|
||||
return Path(path).read_text(encoding='utf-8').strip()
|
||||
url = os.environ.get('DATABASE_URL', '').strip()
|
||||
if not url:
|
||||
raise RuntimeError('DATABASE_URL or DATABASE_URL_FILE is required')
|
||||
return url
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, pool: ConnectionPool):
|
||||
self.pool = pool
|
||||
|
||||
@classmethod
|
||||
def connect(cls):
|
||||
pool = ConnectionPool(
|
||||
conninfo=read_database_url(),
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
timeout=5.0,
|
||||
kwargs={'row_factory': dict_row},
|
||||
open=True,
|
||||
)
|
||||
store = cls(pool)
|
||||
store.run_migrations()
|
||||
return store
|
||||
|
||||
def close(self):
|
||||
self.pool.close()
|
||||
|
||||
def ping(self):
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute('SELECT 1')
|
||||
|
||||
def run_migrations(self):
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
'CREATE TABLE IF NOT EXISTS schema_migrations '
|
||||
'(version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())'
|
||||
)
|
||||
for path in sorted(MIGRATIONS_DIR.glob('*.sql')):
|
||||
version = path.name
|
||||
row = conn.execute(
|
||||
'SELECT 1 FROM schema_migrations WHERE version = %s',
|
||||
(version,),
|
||||
).fetchone()
|
||||
if row:
|
||||
continue
|
||||
sql = path.read_text(encoding='utf-8')
|
||||
with conn.transaction():
|
||||
conn.execute(sql)
|
||||
conn.execute(
|
||||
'INSERT INTO schema_migrations (version) VALUES (%s)',
|
||||
(version,),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _user_row(row):
|
||||
return {
|
||||
'id': str(row['id']),
|
||||
'username': row['username'],
|
||||
'display_name': row['display_name'],
|
||||
'created_at': row['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'),
|
||||
}
|
||||
|
||||
def register_user(self, username, display_name, password_hash, created_at):
|
||||
user_id = str(uuid.uuid4())
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
'INSERT INTO users (id, username, display_name, password_hash, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
(user_id, username, display_name, password_hash, created_at),
|
||||
)
|
||||
except errors.UniqueViolation:
|
||||
raise ApiError(409, 'conflict', 'Username already exists') from None
|
||||
return {
|
||||
'id': user_id,
|
||||
'username': username,
|
||||
'display_name': display_name,
|
||||
'created_at': created_at,
|
||||
}
|
||||
|
||||
def get_user(self, user_id):
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
'SELECT id, username, display_name, created_at FROM users WHERE id = %s',
|
||||
(user_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return self._user_row(row)
|
||||
|
||||
def get_user_with_password(self, username):
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
'SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s',
|
||||
(username,),
|
||||
).fetchone()
|
||||
return row
|
||||
|
||||
def create_chat(self, user_id, title, member_ids, created_at):
|
||||
chat_id = str(uuid.uuid4())
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
rows = conn.execute(
|
||||
'SELECT id FROM users WHERE id = ANY(%s)',
|
||||
(member_ids,),
|
||||
).fetchall()
|
||||
if len(rows) != len(set(member_ids)):
|
||||
raise ApiError(400, 'invalid_request', 'Unknown chat member')
|
||||
conn.execute(
|
||||
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
|
||||
(chat_id, title, created_at),
|
||||
)
|
||||
for index, member_id in enumerate(member_ids):
|
||||
conn.execute(
|
||||
'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)',
|
||||
(chat_id, member_id, index),
|
||||
)
|
||||
except ApiError:
|
||||
raise
|
||||
except errors.Error:
|
||||
raise ApiError(500, 'internal_error', 'Internal server error') from None
|
||||
return {
|
||||
'id': chat_id,
|
||||
'title': title,
|
||||
'member_ids': list(member_ids),
|
||||
'created_at': created_at,
|
||||
}
|
||||
|
||||
def _is_member(self, conn, chat_id, user_id):
|
||||
row = conn.execute(
|
||||
'SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s',
|
||||
(chat_id, user_id),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def get_chat(self, user_id, chat_id):
|
||||
with self.pool.connection() as conn:
|
||||
if not self._is_member(conn, chat_id, user_id):
|
||||
raise ApiError(404, 'not_found', 'Chat not found')
|
||||
chat = conn.execute(
|
||||
'SELECT id, title, created_at FROM chats WHERE id = %s',
|
||||
(chat_id,),
|
||||
).fetchone()
|
||||
members = conn.execute(
|
||||
'SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY position',
|
||||
(chat_id,),
|
||||
).fetchall()
|
||||
return {
|
||||
'id': str(chat['id']),
|
||||
'title': chat['title'],
|
||||
'member_ids': [str(row['user_id']) for row in members],
|
||||
'created_at': chat['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'),
|
||||
}
|
||||
|
||||
def list_chats(self, user_id, limit, cursor_key):
|
||||
query = (
|
||||
'SELECT c.id, c.title, c.created_at '
|
||||
'FROM chats c '
|
||||
'JOIN chat_members cm ON cm.chat_id = c.id AND cm.user_id = %s '
|
||||
)
|
||||
params = [user_id]
|
||||
if cursor_key:
|
||||
query += 'WHERE (c.created_at, c.id) > (%s, %s) '
|
||||
params.extend(cursor_key)
|
||||
query += 'ORDER BY c.created_at ASC, c.id ASC LIMIT %s'
|
||||
params.append(limit + 1)
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(query, params).fetchall()
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
items = []
|
||||
for row in page:
|
||||
chat_id = str(row['id'])
|
||||
items.append(self.get_chat(user_id, chat_id))
|
||||
next_cursor = None
|
||||
if has_more:
|
||||
last = page[-1]
|
||||
next_cursor = (last['created_at'], str(last['id']))
|
||||
return items, next_cursor
|
||||
|
||||
def create_message(self, user_id, chat_id, text, created_at):
|
||||
message_id = str(uuid.uuid4())
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
if not self._is_member(conn, chat_id, user_id):
|
||||
raise ApiError(404, 'not_found', 'Chat not found')
|
||||
conn.execute(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
(message_id, chat_id, user_id, text, created_at),
|
||||
)
|
||||
return {
|
||||
'id': message_id,
|
||||
'chat_id': chat_id,
|
||||
'sender_id': user_id,
|
||||
'text': text,
|
||||
'created_at': created_at,
|
||||
}
|
||||
|
||||
def list_messages(self, user_id, chat_id, limit, cursor_key):
|
||||
with self.pool.connection() as conn:
|
||||
if not self._is_member(conn, chat_id, user_id):
|
||||
raise ApiError(404, 'not_found', 'Chat not found')
|
||||
query = (
|
||||
'SELECT id, chat_id, sender_id, text, created_at FROM messages '
|
||||
'WHERE chat_id = %s '
|
||||
)
|
||||
params = [chat_id]
|
||||
if cursor_key:
|
||||
query += 'AND (created_at, id) > (%s, %s) '
|
||||
params.extend(cursor_key)
|
||||
query += 'ORDER BY created_at ASC, id ASC LIMIT %s'
|
||||
params.append(limit + 1)
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(query, params).fetchall()
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
items = [
|
||||
{
|
||||
'id': str(row['id']),
|
||||
'chat_id': str(row['chat_id']),
|
||||
'sender_id': str(row['sender_id']),
|
||||
'text': row['text'],
|
||||
'created_at': row['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'),
|
||||
}
|
||||
for row in page
|
||||
]
|
||||
next_cursor = None
|
||||
if has_more:
|
||||
last = page[-1]
|
||||
next_cursor = (last['created_at'], str(last['id']))
|
||||
return items, next_cursor
|
||||
@@ -0,0 +1,13 @@
|
||||
EXPLAIN (ANALYZE, BUFFERS) on 10_000 messages in one chat.
|
||||
|
||||
Command (from repo root, stack running):
|
||||
docker compose run --rm \
|
||||
-e DATABASE_URL=postgresql://messenger:messenger_dev_only@postgres:5432/messenger \
|
||||
-e EXPLAIN_MESSAGE_COUNT=10000 \
|
||||
-v ./scripts:/scripts:ro \
|
||||
api python /scripts/explain_messages.py
|
||||
|
||||
Observed plan uses index messages_chat_history_idx (Bitmap Index Scan on chat_id),
|
||||
not Seq Scan on all messages. Execution Time ~2.3 ms on local Docker.
|
||||
|
||||
Keyset pagination in API: WHERE chat_id = $1 AND (created_at, id) > ($cursor...) LIMIT n+1
|
||||
@@ -0,0 +1,13 @@
|
||||
# Track 2 — план запросов и восстановление
|
||||
|
||||
Гипотеза: для skewed-нагрузки (70k сообщений в одном чате) keyset по `(chat_id, created_at, id)` даёт стабильное чтение; большой `OFFSET` и лишний индекс `(chat_id)` ухудшают latency/запись.
|
||||
|
||||
Повтор:
|
||||
|
||||
```sh
|
||||
py -3 scripts/track2_run.py
|
||||
py -3 scripts/track2_backup_restore.py
|
||||
RUN_TRACK2_VERIFY=1 py -3 -m unittest student-tests.test_track2
|
||||
```
|
||||
|
||||
Артефакты: `benchmark.json`, `backup.dump`, `restore_report.json`.
|
||||
Binary file not shown.
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"hypothesis": "Keyset pagination on (chat_id, created_at, id) avoids large offsets; redundant (chat_id)-only index does not improve reads but slows writes.",
|
||||
"hot_chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df",
|
||||
"read_plans": {
|
||||
"keyset_limit_50": {
|
||||
"node_type": "Index Only Scan",
|
||||
"index_name": "messages_chat_history_idx",
|
||||
"execution_ms": 0.349,
|
||||
"plan_rows": 11
|
||||
},
|
||||
"offset_50000_limit_50": {
|
||||
"node_type": "Index Only Scan",
|
||||
"index_name": "messages_chat_history_idx",
|
||||
"execution_ms": 7.043,
|
||||
"plan_rows": 1
|
||||
},
|
||||
"keyset_with_extra_chat_id_index": {
|
||||
"node_type": "Index Scan",
|
||||
"index_name": "messages_chat_id_only_idx",
|
||||
"execution_ms": 9.797,
|
||||
"plan_rows": 11
|
||||
}
|
||||
},
|
||||
"write_messages_per_second": {
|
||||
"with_extra_chat_id_index": 18199.5,
|
||||
"production_indexes_only": 20166.9
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"source": {
|
||||
"messages": 111012,
|
||||
"users": 43,
|
||||
"sample": [
|
||||
{
|
||||
"id": "d4bff878-42c2-4ed2-8261-e2fc42583f65",
|
||||
"chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df",
|
||||
"text": "w"
|
||||
},
|
||||
{
|
||||
"id": "c561eb24-1381-491f-b78c-b9fe460015b5",
|
||||
"chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df",
|
||||
"text": "w"
|
||||
},
|
||||
{
|
||||
"id": "e0789e8a-6c7b-40e9-8141-64d8547de768",
|
||||
"chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df",
|
||||
"text": "w"
|
||||
},
|
||||
{
|
||||
"id": "d0e5a399-932d-4006-91b1-f1cc26501af5",
|
||||
"chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df",
|
||||
"text": "w"
|
||||
},
|
||||
{
|
||||
"id": "de8288c2-a7fb-496b-8be7-cbfe82d138c5",
|
||||
"chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df",
|
||||
"text": "w"
|
||||
}
|
||||
]
|
||||
},
|
||||
"restored": {
|
||||
"messages": 111012,
|
||||
"users": 43,
|
||||
"sample": [
|
||||
{
|
||||
"id": "d4bff878-42c2-4ed2-8261-e2fc42583f65",
|
||||
"chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df",
|
||||
"text": "w"
|
||||
},
|
||||
{
|
||||
"id": "c561eb24-1381-491f-b78c-b9fe460015b5",
|
||||
"chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df",
|
||||
"text": "w"
|
||||
},
|
||||
{
|
||||
"id": "e0789e8a-6c7b-40e9-8141-64d8547de768",
|
||||
"chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df",
|
||||
"text": "w"
|
||||
},
|
||||
{
|
||||
"id": "d0e5a399-932d-4006-91b1-f1cc26501af5",
|
||||
"chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df",
|
||||
"text": "w"
|
||||
},
|
||||
{
|
||||
"id": "de8288c2-a7fb-496b-8be7-cbfe82d138c5",
|
||||
"chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df",
|
||||
"text": "w"
|
||||
}
|
||||
]
|
||||
},
|
||||
"counts_match": true,
|
||||
"sample_match": true,
|
||||
"restore_seconds": 7.09,
|
||||
"dump_bytes": 2899617,
|
||||
"data_loss_boundary": "Logical pg_dump is consistent at dump start; concurrent writes during dump are not included. WAL/PITR not used — RPO equals changes after dump began."
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS users_username_key ON users (username);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
position INT NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS chat_members_user_id_idx ON chat_members (user_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS messages_chat_history_idx ON messages (chat_id, created_at ASC, id ASC);
|
||||
@@ -0,0 +1,3 @@
|
||||
argon2-cffi==23.1.0
|
||||
psycopg[binary,pool]==3.2.9
|
||||
redis==5.2.1
|
||||
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Seed >=10k messages and print EXPLAIN (ANALYZE, BUFFERS) for history pagination."""
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import psycopg
|
||||
|
||||
|
||||
def main():
|
||||
url = os.environ.get('DATABASE_URL')
|
||||
if not url:
|
||||
print('Set DATABASE_URL', file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
chat_id = os.environ.get('EXPLAIN_CHAT_ID')
|
||||
user_id = os.environ.get('EXPLAIN_USER_ID')
|
||||
count = int(os.environ.get('EXPLAIN_MESSAGE_COUNT', '10000'))
|
||||
with psycopg.connect(url) as conn:
|
||||
if not chat_id or not user_id:
|
||||
user_id = str(uuid.uuid4())
|
||||
chat_id = str(uuid.uuid4())
|
||||
now = datetime.now(timezone.utc)
|
||||
conn.execute(
|
||||
'INSERT INTO users (id, username, display_name, password_hash, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
(user_id, 'bench_' + uuid.uuid4().hex[:8], 'Bench', 'x', now),
|
||||
)
|
||||
conn.execute(
|
||||
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
|
||||
(chat_id, 'Bench', now),
|
||||
)
|
||||
conn.execute(
|
||||
'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, 0)',
|
||||
(chat_id, user_id),
|
||||
)
|
||||
print(f'created chat_id={chat_id} user_id={user_id}')
|
||||
existing = conn.execute(
|
||||
'SELECT count(*) FROM messages WHERE chat_id = %s',
|
||||
(chat_id,),
|
||||
).fetchone()[0]
|
||||
need = max(0, count - existing)
|
||||
base = datetime.now(timezone.utc)
|
||||
with conn.cursor() as cur:
|
||||
for index in range(need):
|
||||
cur.execute(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
(
|
||||
str(uuid.uuid4()),
|
||||
chat_id,
|
||||
user_id,
|
||||
'm',
|
||||
base,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
plan = conn.execute(
|
||||
'EXPLAIN (ANALYZE, BUFFERS) '
|
||||
'SELECT id FROM messages WHERE chat_id = %s '
|
||||
'ORDER BY created_at ASC, id ASC LIMIT 50',
|
||||
(chat_id,),
|
||||
).fetchall()
|
||||
for row in plan:
|
||||
print(row[0])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
docker compose restart api postgres redis
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Cross-platform restart hook; on Windows use: ACTION_SCRIPT=scripts/restart.py"""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
result = subprocess.run(
|
||||
['docker', 'compose', 'restart', 'api', 'postgres', 'redis'],
|
||||
cwd=root,
|
||||
check=False,
|
||||
)
|
||||
raise SystemExit(result.returncode)
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
root="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
cd "$root"
|
||||
docker compose restart api postgres redis
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""pg_dump → fresh Postgres volume → verification via psql (no local psycopg)."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / 'evidence' / 'track2'
|
||||
DUMP = OUT / 'backup.dump'
|
||||
MAIN = ['docker', 'compose', '-f', 'compose.yaml']
|
||||
RESTORE = ['docker', 'compose', '-f', 'compose.track2-restore.yaml']
|
||||
PSQL = os.environ.get('TRACK2_PSQL_SERVICE', 'postgres')
|
||||
RESTORE_SVC = 'postgres_restore'
|
||||
|
||||
|
||||
def run(cmd, **kwargs):
|
||||
subprocess.run(cmd, cwd=ROOT, check=True, **kwargs)
|
||||
|
||||
|
||||
def psql(service, compose, database, sql):
|
||||
out = subprocess.check_output(
|
||||
compose + ['exec', '-T', service, 'psql', '-U', 'messenger', '-d', database, '-tAc', sql],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
)
|
||||
return out.strip()
|
||||
|
||||
|
||||
def snapshot(service, compose, database):
|
||||
messages = int(psql(service, compose, database, 'SELECT count(*) FROM messages'))
|
||||
users = int(psql(service, compose, database, 'SELECT count(*) FROM users'))
|
||||
raw = subprocess.check_output(
|
||||
compose + [
|
||||
'exec', '-T', service, 'psql', '-U', 'messenger', '-d', database, '-tAc',
|
||||
"SELECT id::text || '|' || chat_id::text || '|' || text FROM messages "
|
||||
'ORDER BY created_at DESC LIMIT 5',
|
||||
],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
).strip()
|
||||
sample = []
|
||||
if raw:
|
||||
for line in raw.splitlines():
|
||||
parts = line.split('|', 2)
|
||||
sample.append({'id': parts[0], 'chat_id': parts[1], 'text': parts[2]})
|
||||
return {'messages': messages, 'users': users, 'sample': sample}
|
||||
|
||||
|
||||
def main():
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
started = time.perf_counter()
|
||||
run(RESTORE + ['up', '-d', '--wait'])
|
||||
with DUMP.open('wb') as handle:
|
||||
run(MAIN + ['exec', '-T', PSQL, 'pg_dump', '-U', 'messenger', '-Fc', 'messenger'], stdout=handle)
|
||||
before = snapshot(PSQL, MAIN, 'messenger')
|
||||
run(
|
||||
RESTORE + ['exec', '-T', RESTORE_SVC, 'psql', '-U', 'messenger', '-d', 'postgres', '-c',
|
||||
'DROP DATABASE IF EXISTS messenger_restore'],
|
||||
)
|
||||
run(
|
||||
RESTORE + ['exec', '-T', RESTORE_SVC, 'psql', '-U', 'messenger', '-d', 'postgres', '-c',
|
||||
'CREATE DATABASE messenger_restore'],
|
||||
)
|
||||
run(RESTORE + ['cp', str(DUMP), f'{RESTORE_SVC}:/tmp/backup.dump'])
|
||||
restore = subprocess.run(
|
||||
RESTORE + [
|
||||
'exec', '-T', RESTORE_SVC,
|
||||
'pg_restore', '-U', 'messenger', '-d', 'messenger_restore',
|
||||
'--no-owner', '--no-privileges', '/tmp/backup.dump',
|
||||
],
|
||||
cwd=ROOT,
|
||||
)
|
||||
if restore.returncode not in (0, 1):
|
||||
restore.check_returncode()
|
||||
after = snapshot(RESTORE_SVC, RESTORE, 'messenger_restore')
|
||||
elapsed = time.perf_counter() - started
|
||||
ok = (
|
||||
before['messages'] == after['messages']
|
||||
and before['users'] == after['users']
|
||||
and before['sample'] == after['sample']
|
||||
)
|
||||
report = {
|
||||
'source': before,
|
||||
'restored': after,
|
||||
'counts_match': before['messages'] == after['messages'],
|
||||
'sample_match': before['sample'] == after['sample'],
|
||||
'restore_seconds': round(elapsed, 2),
|
||||
'dump_bytes': DUMP.stat().st_size,
|
||||
'data_loss_boundary': (
|
||||
'Logical pg_dump is consistent at dump start; concurrent writes during dump are not '
|
||||
'included. WAL/PITR not used — RPO equals changes after dump began.'
|
||||
),
|
||||
}
|
||||
(OUT / 'restore_report.json').write_text(
|
||||
json.dumps(report, indent=2, ensure_ascii=False),
|
||||
encoding='utf-8',
|
||||
)
|
||||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
if not ok:
|
||||
raise SystemExit(1)
|
||||
print('PASS: restore verification')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare index plans and INSERT cost; write evidence/track2/benchmark.json."""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = Path(os.environ.get('TRACK2_EVIDENCE_DIR', str(ROOT / 'evidence' / 'track2')))
|
||||
DATABASE_URL = os.environ.get('DATABASE_URL', '')
|
||||
HOT_CHAT_TITLE = os.environ.get('TRACK2_HOT_CHAT_TITLE', 'track2_hot')
|
||||
|
||||
|
||||
def _find_index(plan):
|
||||
if plan.get('Index Name'):
|
||||
return plan.get('Index Name'), plan.get('Node Type')
|
||||
for child in plan.get('Plans') or []:
|
||||
found = _find_index(child)
|
||||
if found:
|
||||
return found
|
||||
return None, plan.get('Node Type')
|
||||
|
||||
|
||||
def explain(conn, sql, params):
|
||||
rows = conn.execute('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ' + sql, params).fetchone()[0]
|
||||
plan = rows[0]['Plan']
|
||||
index_name, node_type = _find_index(plan)
|
||||
return {
|
||||
'node_type': node_type,
|
||||
'index_name': index_name,
|
||||
'execution_ms': rows[0]['Execution Time'],
|
||||
'plan_rows': plan.get('Plan Rows'),
|
||||
}
|
||||
|
||||
|
||||
def insert_batch(conn, chat_id, user_id, n):
|
||||
base = datetime.now(timezone.utc)
|
||||
start = time.perf_counter()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
[
|
||||
(str(uuid.uuid4()), chat_id, user_id, 'w', base)
|
||||
for _ in range(n)
|
||||
],
|
||||
)
|
||||
conn.commit()
|
||||
elapsed = time.perf_counter() - start
|
||||
return n / elapsed
|
||||
|
||||
|
||||
def main():
|
||||
if not DATABASE_URL:
|
||||
print('DATABASE_URL required', file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
with psycopg.connect(DATABASE_URL) as conn:
|
||||
hot = conn.execute(
|
||||
'SELECT c.id FROM chats c WHERE c.title = %s',
|
||||
(HOT_CHAT_TITLE,),
|
||||
).fetchone()
|
||||
if not hot:
|
||||
print('Run track2_seed.py first', file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
hot_id = str(hot[0])
|
||||
user = conn.execute(
|
||||
"SELECT id FROM users WHERE username = 'track2_bench'"
|
||||
).fetchone()
|
||||
user_id = str(user[0])
|
||||
keyset_sql = (
|
||||
'SELECT id FROM messages WHERE chat_id = %s '
|
||||
'ORDER BY created_at ASC, id ASC LIMIT 50'
|
||||
)
|
||||
offset_sql = (
|
||||
'SELECT id FROM messages WHERE chat_id = %s '
|
||||
'ORDER BY created_at ASC, id ASC OFFSET 50000 LIMIT 50'
|
||||
)
|
||||
plan_keyset = explain(conn, keyset_sql, (hot_id,))
|
||||
plan_offset = explain(conn, offset_sql, (hot_id,))
|
||||
conn.execute('DROP INDEX IF EXISTS messages_chat_id_only_idx')
|
||||
conn.execute('CREATE INDEX messages_chat_id_only_idx ON messages (chat_id)')
|
||||
conn.commit()
|
||||
plan_keyset_with_extra = explain(conn, keyset_sql, (hot_id,))
|
||||
write_before = insert_batch(conn, hot_id, user_id, 500)
|
||||
conn.execute('DROP INDEX IF EXISTS messages_chat_id_only_idx')
|
||||
conn.commit()
|
||||
write_after = insert_batch(conn, hot_id, user_id, 500)
|
||||
result = {
|
||||
'hypothesis': (
|
||||
'Keyset pagination on (chat_id, created_at, id) avoids large offsets; '
|
||||
'redundant (chat_id)-only index does not improve reads but slows writes.'
|
||||
),
|
||||
'hot_chat_id': hot_id,
|
||||
'read_plans': {
|
||||
'keyset_limit_50': plan_keyset,
|
||||
'offset_50000_limit_50': plan_offset,
|
||||
'keyset_with_extra_chat_id_index': plan_keyset_with_extra,
|
||||
},
|
||||
'write_messages_per_second': {
|
||||
'with_extra_chat_id_index': round(write_before, 1),
|
||||
'production_indexes_only': round(write_after, 1),
|
||||
},
|
||||
}
|
||||
out_path = OUT / 'benchmark.json'
|
||||
out_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding='utf-8')
|
||||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||||
print('wrote', out_path)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run full track 2 pipeline (seed, benchmark, backup/restore)."""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DB = 'postgresql://messenger:messenger_dev_only@postgres:5432/messenger'
|
||||
EVIDENCE = ROOT / 'evidence' / 'track2'
|
||||
EVIDENCE.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def docker_python(script):
|
||||
subprocess.run(
|
||||
[
|
||||
'docker', 'compose', 'run', '--rm',
|
||||
'-e', 'DATABASE_URL=' + DB,
|
||||
'-e', 'TRACK2_EVIDENCE_DIR=/evidence',
|
||||
'-v', f'{ROOT / "scripts"}:/scripts:ro',
|
||||
'-v', f'{EVIDENCE}:/evidence',
|
||||
'api', 'python', f'/scripts/{script}',
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
for name in ('track2_seed.py', 'track2_benchmark.py'):
|
||||
print('===', name, '===')
|
||||
docker_python(name)
|
||||
print('=== track2_backup_restore.py ===')
|
||||
subprocess.run([sys.executable, str(ROOT / 'scripts' / 'track2_backup_restore.py')], cwd=ROOT, check=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Seed >=100k messages with skewed chat sizes (track 2)."""
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import psycopg
|
||||
|
||||
TARGET = int(os.environ.get('TRACK2_MESSAGE_COUNT', '100000'))
|
||||
DATABASE_URL = os.environ.get('DATABASE_URL', '')
|
||||
|
||||
|
||||
def main():
|
||||
if not DATABASE_URL:
|
||||
print('DATABASE_URL required', file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
base = datetime.now(timezone.utc)
|
||||
with psycopg.connect(DATABASE_URL) as conn:
|
||||
hot_exists = conn.execute(
|
||||
"SELECT 1 FROM chats WHERE title = 'track2_hot'"
|
||||
).fetchone()
|
||||
if hot_exists:
|
||||
total = conn.execute('SELECT count(*) FROM messages').fetchone()[0]
|
||||
print(f'skip_seed existing track2 dataset messages_total={total}')
|
||||
return
|
||||
row = conn.execute(
|
||||
"SELECT id FROM users WHERE username = 'track2_bench'"
|
||||
).fetchone()
|
||||
if row:
|
||||
user_id = str(row[0])
|
||||
else:
|
||||
user_id = str(uuid.uuid4())
|
||||
conn.execute(
|
||||
'INSERT INTO users (id, username, display_name, password_hash, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
(user_id, 'track2_bench', 'Track2', 'x', base),
|
||||
)
|
||||
plans = [
|
||||
('hot', int(TARGET * 0.70)),
|
||||
('medium_a', int(TARGET * 0.05)),
|
||||
('medium_b', int(TARGET * 0.05)),
|
||||
('medium_c', int(TARGET * 0.05)),
|
||||
('medium_d', int(TARGET * 0.05)),
|
||||
('medium_e', int(TARGET * 0.05)),
|
||||
]
|
||||
tail = TARGET - sum(count for _, count in plans)
|
||||
small_chats = 50
|
||||
per_small = max(1, tail // small_chats)
|
||||
chat_ids = []
|
||||
for name, count in plans:
|
||||
chat_id = str(uuid.uuid4())
|
||||
chat_ids.append((chat_id, count, name))
|
||||
conn.execute(
|
||||
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
|
||||
(chat_id, 'track2_' + name, base),
|
||||
)
|
||||
conn.execute(
|
||||
'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, 0)',
|
||||
(chat_id, user_id),
|
||||
)
|
||||
for index in range(small_chats):
|
||||
chat_id = str(uuid.uuid4())
|
||||
chat_ids.append((chat_id, per_small, f'small_{index}'))
|
||||
conn.execute(
|
||||
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
|
||||
(chat_id, 'track2_small_' + str(index), base),
|
||||
)
|
||||
conn.execute(
|
||||
'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, 0)',
|
||||
(chat_id, user_id),
|
||||
)
|
||||
total = 0
|
||||
with conn.cursor() as cur:
|
||||
for chat_id, count, _name in chat_ids:
|
||||
rows = []
|
||||
for offset in range(count):
|
||||
rows.append(
|
||||
(
|
||||
str(uuid.uuid4()),
|
||||
chat_id,
|
||||
user_id,
|
||||
't',
|
||||
base + timedelta(microseconds=offset),
|
||||
)
|
||||
)
|
||||
cur.executemany(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
rows,
|
||||
)
|
||||
total += count
|
||||
conn.commit()
|
||||
actual = conn.execute('SELECT count(*) FROM messages').fetchone()[0]
|
||||
hot = conn.execute(
|
||||
"SELECT count(*) FROM messages m JOIN chats c ON c.id = m.chat_id WHERE c.title = 'track2_hot'"
|
||||
).fetchone()[0]
|
||||
print(f'seeded_messages={total} table_total={actual} hot_chat_messages={hot}')
|
||||
print(f'benchmark_user_id={user_id}')
|
||||
for chat_id, count, name in chat_ids[:3]:
|
||||
print(f'sample_chat {name} id={chat_id} messages={count}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
import os
|
||||
|
||||
import redis
|
||||
|
||||
|
||||
class SessionStore:
|
||||
def __init__(self, url, connect_timeout=2.0, socket_timeout=2.0):
|
||||
self._url = url
|
||||
self._client = redis.from_url(
|
||||
url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=connect_timeout,
|
||||
socket_timeout=socket_timeout,
|
||||
health_check_interval=30,
|
||||
)
|
||||
self._prefix = 'sess:'
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
url = _read_config('CACHE_URL_FILE', 'CACHE_URL')
|
||||
if not url:
|
||||
raise RuntimeError('CACHE_URL or CACHE_URL_FILE is required when SESSION_STORE=redis')
|
||||
return cls(url)
|
||||
|
||||
def ping(self):
|
||||
return self._client.ping()
|
||||
|
||||
def create(self, token, user_id, ttl_seconds):
|
||||
self._client.setex(self._prefix + token, ttl_seconds, user_id)
|
||||
|
||||
def get_user_id(self, token):
|
||||
return self._client.get(self._prefix + token)
|
||||
|
||||
def revoke(self, token):
|
||||
return self._client.delete(self._prefix + token) > 0
|
||||
|
||||
|
||||
def _read_config(file_env, value_env):
|
||||
path = os.environ.get(file_env)
|
||||
if path and os.path.isfile(path):
|
||||
with open(path, encoding='utf-8') as handle:
|
||||
return handle.read().strip()
|
||||
return os.environ.get(value_env, '').strip()
|
||||
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверка отказа Redis/PostgreSQL (уровень 4). Требует Docker и RUN_INFRA_TEST=1."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
|
||||
def compose_argv():
|
||||
override = os.environ.get('COMPOSE', '').strip()
|
||||
if override:
|
||||
return override.split()
|
||||
return ['docker', 'compose']
|
||||
|
||||
|
||||
def compose(*args):
|
||||
result = subprocess.run(
|
||||
[*compose_argv(), *args],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(
|
||||
COMPOSE + ' ' + ' '.join(args) + ' failed: ' + (result.stderr or result.stdout)
|
||||
)
|
||||
|
||||
|
||||
def http(method, path, token=None, expected=None):
|
||||
headers = {'Accept': 'application/json'}
|
||||
if token:
|
||||
headers['Authorization'] = 'Bearer ' + token
|
||||
req = urllib.request.Request(BASE + path, headers=headers, method=method)
|
||||
try:
|
||||
response = urllib.request.urlopen(req, timeout=5)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
body = response.read()
|
||||
payload = json.loads(body) if body else None
|
||||
if expected is not None:
|
||||
if isinstance(expected, int):
|
||||
expected = [expected]
|
||||
if response.code not in expected:
|
||||
raise AssertionError(
|
||||
f'{method} {path}: expected {expected}, got {response.code} {payload}'
|
||||
)
|
||||
return response.code, payload
|
||||
|
||||
|
||||
def wait_ready(deadline=60):
|
||||
end = time.monotonic() + deadline
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
code, _ = http('GET', '/health/ready')
|
||||
if code == 200:
|
||||
return
|
||||
except (urllib.error.URLError, TimeoutError, ConnectionError, json.JSONDecodeError):
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError('API did not become ready')
|
||||
|
||||
|
||||
@unittest.skipUnless(
|
||||
os.environ.get('RUN_INFRA_TEST') == '1',
|
||||
'Set RUN_INFRA_TEST=1 to run Docker dependency failure tests',
|
||||
)
|
||||
class DependencyFailureTests(unittest.TestCase):
|
||||
token = None
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
wait_ready()
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client # noqa: WPS433
|
||||
|
||||
actor = Client(BASE).user('dep')
|
||||
cls.token = actor['token']
|
||||
|
||||
def tearDown(self):
|
||||
compose('start', 'redis', 'postgres')
|
||||
wait_ready()
|
||||
|
||||
def test_redis_down_ready_and_auth(self):
|
||||
compose('stop', 'redis')
|
||||
time.sleep(1)
|
||||
code, payload = http('GET', '/health/ready')
|
||||
self.assertEqual(code, 503)
|
||||
self.assertEqual(payload['error']['code'], 'unavailable')
|
||||
code, payload = http('GET', '/api/v1/users/me', token=self.token)
|
||||
self.assertEqual(code, 503)
|
||||
self.assertEqual(payload['error']['code'], 'unavailable')
|
||||
|
||||
def test_postgres_down_ready(self):
|
||||
compose('stop', 'postgres')
|
||||
time.sleep(1)
|
||||
code, payload = http('GET', '/health/ready')
|
||||
self.assertEqual(code, 503)
|
||||
self.assertEqual(payload['error']['code'], 'unavailable')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Дополнительные проверки уровня 4: auth, expiry, гонка username, rollback чата."""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
|
||||
BASE = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
|
||||
|
||||
|
||||
def raw_request(method, path, expected, body=None, headers=None):
|
||||
data = None
|
||||
h = {'Accept': 'application/json'}
|
||||
if body is not None:
|
||||
data = json.dumps(body, ensure_ascii=False).encode('utf-8')
|
||||
h['Content-Type'] = 'application/json'
|
||||
h.update(headers or {})
|
||||
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
|
||||
try:
|
||||
response = urllib.request.urlopen(req, timeout=10)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
payload = response.read()
|
||||
check(response.code == expected, f'{method} {path}: expected {expected}, got {response.code}')
|
||||
return response.code, response.headers, json.loads(payload) if payload else None
|
||||
|
||||
|
||||
class Lab2Grade4Tests(unittest.TestCase):
|
||||
def test_logout_revokes_session(self):
|
||||
c = Client(BASE)
|
||||
actor = c.user('logout')
|
||||
c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
||||
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
|
||||
c.call('GET', '/api/v1/users/me', 401, actor=actor)
|
||||
|
||||
def test_wrong_password_rejected(self):
|
||||
c = Client(BASE)
|
||||
actor = c.user('pwd')
|
||||
bad = base64.b64encode(
|
||||
(actor['registration']['username'] + ':wrong-password-x').encode()
|
||||
).decode()
|
||||
c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': 'Basic ' + bad})
|
||||
|
||||
def test_readiness_requires_dependencies(self):
|
||||
c = Client(BASE)
|
||||
c.call('GET', '/health/ready', schema='Health')
|
||||
|
||||
@unittest.skipUnless(
|
||||
os.environ.get('RUN_EXPIRY_TEST') == '1',
|
||||
'Set RUN_EXPIRY_TEST=1 and SESSION_TTL_SECONDS=2 on the API, then rerun',
|
||||
)
|
||||
def test_session_expiry(self):
|
||||
c = Client(BASE)
|
||||
username = 'u_exp_' + uuid.uuid4().hex[:12]
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
body = {'username': username, 'display_name': 'Expiry', 'password': password}
|
||||
user = c.call('POST', '/api/v1/users', 201, body=body, schema='User')['json']
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
session = c.call(
|
||||
'POST', '/api/v1/auth/sessions', 201,
|
||||
headers={'Authorization': 'Basic ' + credential},
|
||||
schema='Session',
|
||||
)['json']
|
||||
actor = {'id': user['id'], 'user': user, 'registration': body, 'token': session['token']}
|
||||
time.sleep(3)
|
||||
c.call('GET', '/api/v1/users/me', 401, actor=actor)
|
||||
|
||||
def test_duplicate_username_race(self):
|
||||
c = Client(BASE)
|
||||
username = 'u_race_' + uuid.uuid4().hex[:10]
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
body = {
|
||||
'username': username,
|
||||
'display_name': 'Race',
|
||||
'password': password,
|
||||
}
|
||||
|
||||
def register():
|
||||
data = json.dumps(body, ensure_ascii=False).encode('utf-8')
|
||||
req = urllib.request.Request(
|
||||
BASE + '/api/v1/users',
|
||||
data=data,
|
||||
headers={'Accept': 'application/json', 'Content-Type': 'application/json'},
|
||||
method='POST',
|
||||
)
|
||||
try:
|
||||
response = urllib.request.urlopen(req, timeout=10)
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code
|
||||
with response:
|
||||
return response.code
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
statuses = list(pool.map(lambda _: register(), range(8)))
|
||||
check(all(code in (201, 409) for code in statuses), f'unexpected race statuses: {set(statuses)}')
|
||||
check(sum(1 for code in statuses if code == 201) == 1, 'expected exactly one 201')
|
||||
|
||||
def test_chat_creation_rollback_unknown_member(self):
|
||||
c = Client(BASE)
|
||||
alice = c.user('rb_alice')
|
||||
bob = c.user('rb_bob')
|
||||
fake = str(uuid.uuid4())
|
||||
c.call(
|
||||
'POST', '/api/v1/chats', 400, actor=alice,
|
||||
body={'title': 'Bad', 'member_ids': [alice['id'], fake]},
|
||||
)
|
||||
listed = c.call('GET', '/api/v1/chats?limit=100', actor=alice, schema='ChatPage')['json']
|
||||
check(all(item['title'] != 'Bad' for item in listed['items']), 'chat row leaked after rollback')
|
||||
listed_bob = c.call('GET', '/api/v1/chats?limit=100', actor=bob, schema='ChatPage')['json']
|
||||
check(all(item['title'] != 'Bad' for item in listed_bob['items']), 'chat row leaked to bob')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sanity checks for track 2 evidence artifacts."""
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / 'evidence' / 'track2'
|
||||
|
||||
|
||||
@unittest.skipUnless(
|
||||
os.environ.get('RUN_TRACK2_VERIFY') == '1',
|
||||
'Set RUN_TRACK2_VERIFY=1 after running track2 scripts',
|
||||
)
|
||||
class Track2EvidenceTests(unittest.TestCase):
|
||||
def test_benchmark_artifact(self):
|
||||
path = EVIDENCE / 'benchmark.json'
|
||||
self.assertTrue(path.is_file(), 'missing benchmark.json — run scripts/track2_benchmark.py')
|
||||
data = json.loads(path.read_text(encoding='utf-8'))
|
||||
keyset = data['read_plans']['keyset_limit_50']['execution_ms']
|
||||
offset = data['read_plans']['offset_50000_limit_50']['execution_ms']
|
||||
self.assertLess(keyset, offset, 'keyset should be faster than large OFFSET on hot chat')
|
||||
self.assertIsNotNone(data['read_plans']['keyset_limit_50'].get('index_name'))
|
||||
|
||||
def test_restore_artifact(self):
|
||||
path = EVIDENCE / 'restore_report.json'
|
||||
self.assertTrue(path.is_file(), 'missing restore_report.json')
|
||||
data = json.loads(path.read_text(encoding='utf-8'))
|
||||
self.assertTrue(data['counts_match'])
|
||||
self.assertTrue(data['sample_match'])
|
||||
self.assertGreaterEqual(data['source']['messages'], 100_000)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user