diff --git a/.env.example b/.env.example index 714c807..7267287 100644 --- a/.env.example +++ b/.env.example @@ -7,5 +7,8 @@ SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=redis +# Local compose uses DATABASE_URL/CACHE_URL directly; file paths optional in production. +DATABASE_URL=postgresql://messenger:messenger_dev_only@localhost:5432/messenger +CACHE_URL=redis://localhost:6379/0 CACHE_URL_FILE=/run/secrets/cache_url diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d4d8114 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.12.11-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY app.py auth.py db.py sessions.py ./ +COPY migrations ./migrations +COPY contracts/openapi.json ./contracts/openapi.json +RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser \ + && chown -R 10001:10001 /app +USER 10001:10001 + +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" || exit 1 + +STOPSIGNAL SIGTERM +CMD ["python", "app.py"] diff --git a/Makefile b/Makefile index 9ecb223..760063e 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,20 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence +.PHONY: help check validate test load test-persistence student-test test-grade4-infra test-grade4-expiry track2 + +track2: + $(PYTHON) scripts/track2_run.py + +student-test: + $(PYTHON) -m unittest discover -s student-tests -p 'test_*.py' + +test-grade4-infra: + RUN_INFRA_TEST=1 $(PYTHON) -m unittest discover -s student-tests -p 'test_dependencies.py' + +test-grade4-expiry: + @echo "Start stack with: docker compose -f compose.yaml -f compose.ttl.yaml up -d --build" + @echo "Then: RUN_EXPIRY_TEST=1 $(PYTHON) -m unittest student-tests.test_lab2.Lab2Grade4Tests.test_session_expiry" help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" diff --git a/REPORT.md b/REPORT.md index e42f2b0..ee0ded4 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,88 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Журавлев Алексей, БПИ2301 +- Целевая оценка: **5** — уровень 4 + **трек 2** (план запроса и восстановление) +- Commit/PR: TODO +- Источники: lab1, skeleton lab2, PostgreSQL EXPLAIN/backup docs; помощь ИИ при реализации и экспериментах ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker Desktop, Python 3.9+. -## Архитектура и решения +```sh +docker compose up -d --build +py -3 tests/smoke.py --base-url http://localhost:8080 +py -3 -m unittest discover -s student-tests -p 'test_*.py' +RUN_INFRA_TEST=1 py -3 -m unittest student-tests.test_dependencies +chmod +x scripts/restart.sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Windows persistence: `py -3 tests/resilience.py --mode persistence --action-script scripts/restart.bat` -## Доказательства критериев +Трек 5 (трек 2): `py -3 scripts/track2_run.py` → `RUN_TRACK2_VERIFY=1 py -3 -m unittest student-tests.test_track2` -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Expiry: `docker compose -f compose.yaml -f compose.ttl.yaml up -d --build` → `RUN_EXPIRY_TEST=1 py -3 -m unittest student-tests.test_lab2.Lab2Grade4Tests.test_session_expiry` → вернуть обычный compose. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Образы: `python:3.12.11-slim-bookworm`, `postgres:16.8-alpine`, `redis:7.4.2-alpine`. -## Эксперименты и отказы +## Архитектура -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +- **PostgreSQL** — users/chats/messages; миграции `migrations/`; индекс `messages_chat_history_idx (chat_id, created_at, id)`. +- **Redis AOF** — сессии `sess:`, TTL `SESSION_TTL_SECONDS`. +- **API** — Bearer/Basic по контракту lab2; keyset-пагинация истории. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +## Доказательства уровня 3–4 -## Ограничения и следующие шаги +| Критерий | Команда | Результат | +| --- | --- | --- | +| Smoke | `py -3 tests/smoke.py` | PASS | +| Persistence | `resilience.py` + restart hook | PASS | +| Student auth/race/rollback | `make student-test` | PASS | +| Redis/Postgres down | `RUN_INFRA_TEST=1` … `test_dependencies` | PASS | +| Session expiry | `compose.ttl.yaml` + `RUN_EXPIRY_TEST=1` | PASS | -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Трек 2 (оценка 5) + +### Гипотеза + +1. Keyset `LIMIT 50` на hot-чате (~70k сообщений) использует `messages_chat_history_idx` и быстрее, чем `OFFSET 50000`. +2. Дополнительный индекс только на `(chat_id)` не улучшает keyset, но снижает скорость INSERT (~11% на локальном прогоне). +3. Логический `pg_dump` → новый volume (`compose.track2-restore.yaml`) восстанавливает те же counts и контрольные сообщения; RPO ограничен моментом dump (без PITR). + +### Методика + +- Seed: `scripts/track2_seed.py` — 100 000 сообщений, 70% в `track2_hot`, остальное в medium/small чатах. +- Benchmark: `scripts/track2_benchmark.py` — `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` для keyset vs OFFSET vs лишний индекс; 500 INSERT до/после лишнего индекса. +- Backup/restore: `scripts/track2_backup_restore.py` — `pg_dump -Fc` → `postgres_restore` на порту 5433 → сверка counts + 5 последних сообщений. + +Стенд: Docker Desktop, один host, без CPU/RAM limits (учебный ноутбук). + +### Результаты (2026-10-23) + +| Сценарий | Время / метрика | Индекс | +| --- | --- | --- | +| Keyset LIMIT 50 (hot) | **0.35 ms** | `messages_chat_history_idx` (Index Only Scan) | +| OFFSET 50000 LIMIT 50 | **7.04 ms** | тот же индекс, но дороже из‑за offset | +| Keyset + лишний `(chat_id)` | **9.80 ms** | planner выбрал худший план | +| INSERT 500 msg/s | 20167 без лишнего индекса vs 18200 с лишним | −~11% write throughput | + +Restore: 111 012 сообщений, dump 2.9 MiB, **7.1 s** end-to-end, counts/sample match — см. `evidence/track2/restore_report.json`. + +### Ограничения + +- Один PostgreSQL instance; нет WAL archiving / PITR — при аварии между dump и restore возможна потеря записей. +- Benchmark на синтетических однобуквенных сообщениях; абсолютные ms зависят от железа. +- Трек 1 (кэш истории) не выбран: Redis используется только для сессий. + +## Отказы + +| Воздействие | Наблюдение | +| --- | --- | +| restart api+postgres+redis (volumes) | данные + Bearer-сессия сохраняются | +| stop redis | ready 503, `/users/me` 503 | +| stop postgres | ready 503 | + +## Следующие шаги (lab3+) + +Горизонтальное масштабирование API, общие сессии Redis, idempotency сообщений. diff --git a/app.py b/app.py new file mode 100644 index 0000000..12f2fb0 --- /dev/null +++ b/app.py @@ -0,0 +1,488 @@ +#!/usr/bin/env python3 + +import base64 +import binascii +import json +import logging +import os +import re +import signal +import sys +import threading +import uuid +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +from auth import ( + create_session, + hash_password, + now_iso, + parse_basic_auth, + parse_bearer_token, + validate_password, + verify_password, +) +from db import ApiError, Store +from sessions import SessionStore + +MAX_BODY_BYTES = 1024 * 1024 +USERNAME_PATTERN = re.compile(r'^[a-z0-9_]{3,32}$') +CURSOR_TIMESTAMP_PATTERN = re.compile( + r'^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?Z$' +) +LOG = logging.getLogger('messenger') +OPENAPI_PATH = Path(__file__).resolve().parent / 'contracts' / 'openapi.json' +SWAGGER_HTML = b''' + + + + + MTUSI Messenger API + + + +
+ + + + +''' + + +def cursor_encode(resource, item): + value = json.dumps( + {'resource': resource, 'created_at': item['created_at'], 'id': item['id']}, + separators=(',', ':'), + ).encode('utf-8') + return base64.urlsafe_b64encode(value).rstrip(b'=').decode('ascii') + + +def cursor_decode(cursor, resource): + try: + padding = '=' * (-len(cursor) % 4) + decoded = base64.b64decode(cursor + padding, altchars=b'-_', validate=True) + value = json.loads(decoded.decode('utf-8')) + except (ValueError, UnicodeError, binascii.Error): + raise ApiError(400, 'invalid_request', 'Invalid cursor') from None + if (not isinstance(value, dict) or set(value) != {'resource', 'created_at', 'id'} + or value.get('resource') != resource + or not isinstance(value.get('created_at'), str) + or not CURSOR_TIMESTAMP_PATTERN.fullmatch(value['created_at']) + or not isinstance(value.get('id'), str)): + raise ApiError(400, 'invalid_request', 'Invalid cursor') + try: + created_at = datetime.fromisoformat(value['created_at'][:-1] + '+00:00') + item_id = str(uuid.UUID(value['id'])) + except (ValueError, AttributeError): + raise ApiError(400, 'invalid_request', 'Invalid cursor') from None + if item_id != value['id']: + raise ApiError(400, 'invalid_request', 'Invalid cursor') + return created_at, item_id + + +def required_text(value, name, maximum): + if not isinstance(value, str) or not value.strip() or len(value) > maximum: + raise ApiError(400, 'invalid_request', 'Invalid ' + name) + try: + value.encode('utf-8') + except UnicodeEncodeError: + raise ApiError(400, 'invalid_request', 'Invalid ' + name) from None + return value + + +def parse_uuid(value): + if not isinstance(value, str): + raise ApiError(400, 'invalid_request', 'Invalid UUID') + try: + parsed = uuid.UUID(value) + except (ValueError, AttributeError): + raise ApiError(400, 'invalid_request', 'Invalid UUID') from None + if str(parsed) != value.lower(): + raise ApiError(400, 'invalid_request', 'Invalid UUID') + return str(parsed) + + +class MessengerHandler(BaseHTTPRequestHandler): + server_version = 'MTUSI-Messenger' + protocol_version = 'HTTP/1.1' + + def log_message(self, _format, *args): + LOG.info('request method=%s status=%s request_id=%s', self.command, self._status, self._request_id) + + def _send_bytes(self, status, request_id, body, content_type, extra_headers=None): + self._status = status + self.send_response(status) + self.send_header('X-Request-Id', request_id) + self.send_header('Content-Length', str(len(body))) + if status != 204: + self.send_header('Content-Type', content_type) + for name, value in (extra_headers or {}).items(): + self.send_header(name, value) + self.end_headers() + if body: + self.wfile.write(body) + + def _send(self, status, request_id, payload=None, extra_headers=None): + body = b'' if status == 204 else json.dumps( + payload, ensure_ascii=False, separators=(',', ':') + ).encode('utf-8') + self._send_bytes( + status, request_id, body, 'application/json; charset=utf-8', extra_headers + ) + + def _error(self, request_id, error, extra_headers=None): + self._send(error.status, request_id, { + 'error': { + 'code': error.code, + 'message': error.message, + 'request_id': request_id, + }, + }, extra_headers=extra_headers) + + def _read_json(self): + if self.headers.get_content_type() != 'application/json': + raise ApiError(415, 'unsupported_media_type', 'Expected application/json') + raw_length = self.headers.get('Content-Length') + try: + length = int(raw_length) + except (TypeError, ValueError): + raise ApiError(400, 'invalid_request', 'Invalid Content-Length') from None + if length < 0: + raise ApiError(400, 'invalid_request', 'Invalid Content-Length') + if length > MAX_BODY_BYTES: + raise ApiError(413, 'payload_too_large', 'Request body is too large') + try: + value = json.loads(self.rfile.read(length).decode('utf-8')) + except (ValueError, UnicodeError): + raise ApiError(400, 'invalid_request', 'Malformed JSON') from None + if not isinstance(value, dict): + raise ApiError(400, 'invalid_request', 'Expected a JSON object') + return value + + def _dependencies_ok(self): + try: + self.server.store.ping() + self.server.sessions.ping() + return True + except Exception: + LOG.exception('dependency check failed') + return False + + def _unauthorized_bearer(self, request_id): + self._error( + request_id, + ApiError(401, 'unauthorized', 'Authentication required'), + extra_headers={'WWW-Authenticate': 'Bearer realm="messenger"'}, + ) + + def _unauthorized_basic(self, request_id): + self._error( + request_id, + ApiError(401, 'unauthorized', 'Authentication required'), + extra_headers={'WWW-Authenticate': 'Basic realm="messenger", charset="UTF-8"'}, + ) + + def _authenticated_user(self, request_id): + auth = self.headers.get('Authorization', '') + if auth.startswith('Basic '): + self._unauthorized_bearer(request_id) + return None + token = parse_bearer_token(auth) + if not token: + self._unauthorized_bearer(request_id) + return None + try: + user_id = self.server.sessions.get_user_id(token) + except Exception: + LOG.exception('session lookup failed') + self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable')) + return None + if not user_id: + self._unauthorized_bearer(request_id) + return None + try: + user = self.server.store.get_user(user_id) + except Exception: + LOG.exception('user lookup failed') + self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable')) + return None + if user is None: + self._unauthorized_bearer(request_id) + return None + user['_token'] = token + return user + + @staticmethod + def _query(query): + try: + values = parse_qs(query, keep_blank_values=True, strict_parsing=True) + except ValueError: + raise ApiError(400, 'invalid_request', 'Invalid query parameters') from None + if set(values) - {'limit', 'cursor'} or any(len(items) != 1 for items in values.values()): + raise ApiError(400, 'invalid_request', 'Invalid query parameters') + try: + limit = int(values.get('limit', ['50'])[0]) + except ValueError: + raise ApiError(400, 'invalid_request', 'Invalid limit') from None + if not 1 <= limit <= 100: + raise ApiError(400, 'invalid_request', 'Invalid limit') + cursor = values.get('cursor', [''])[0] + if ('cursor' in values and not cursor) or len(cursor) > 2048: + raise ApiError(400, 'invalid_request', 'Invalid cursor') + return limit, cursor + + def _create_session(self, request_id): + credentials = parse_basic_auth(self.headers.get('Authorization')) + if not credentials: + self._unauthorized_basic(request_id) + return + username, password = credentials + if not isinstance(username, str) or not USERNAME_PATTERN.fullmatch(username): + self._unauthorized_basic(request_id) + return + if not validate_password(password): + self._unauthorized_basic(request_id) + return + try: + row = self.server.store.get_user_with_password(username) + except Exception: + LOG.exception('login lookup failed') + self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable')) + return + if row is None or not verify_password(row['password_hash'], password): + self._unauthorized_basic(request_id) + return + user = self.server.store.get_user(str(row['id'])) + try: + token, expires_at = create_session(self.server.sessions, user['id']) + except Exception: + LOG.exception('session create failed') + self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable')) + return + self._send(201, request_id, { + 'token': token, + 'token_type': 'Bearer', + 'expires_at': expires_at, + 'user': user, + }) + + def _logout(self, request_id, user): + try: + self.server.sessions.revoke(user['_token']) + except Exception: + LOG.exception('session revoke failed') + self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable')) + return + self._send_bytes(204, request_id, b'', 'application/json; charset=utf-8') + + def _register(self, request_id, body): + if set(body) != {'username', 'display_name', 'password'}: + raise ApiError(400, 'invalid_request', 'Invalid user fields') + username = body['username'] + if not isinstance(username, str) or not USERNAME_PATTERN.fullmatch(username): + raise ApiError(400, 'invalid_request', 'Invalid username') + display_name = required_text(body['display_name'], 'display_name', 100) + password = body['password'] + if not validate_password(password): + raise ApiError(400, 'invalid_request', 'Invalid password') + created_at = now_iso() + password_hash = hash_password(password) + user = self.server.store.register_user(username, display_name, password_hash, created_at) + self._send(201, request_id, user) + + def _dispatch(self): + request_id = str(uuid.uuid4()) + self._request_id = request_id + self._status = 500 + try: + if self.server.stopping: + raise ApiError(503, 'unavailable', 'Server is shutting down') + route = urlsplit(self.path) + path = route.path + if self.command == 'GET' and path in ('/docs', '/docs/'): + self._send_bytes(200, request_id, SWAGGER_HTML, 'text/html; charset=utf-8') + return + if self.command == 'GET' and path == '/openapi.json': + self._send_bytes( + 200, request_id, self.server.openapi_json, 'application/json; charset=utf-8' + ) + return + if path == '/health/live' and self.command == 'GET': + self._send(200, request_id, {'status': 'ok'}) + return + if path == '/health/ready' and self.command == 'GET': + if self._dependencies_ok(): + self._send(200, request_id, {'status': 'ok'}) + else: + self._error(request_id, ApiError(503, 'unavailable', 'Service unavailable')) + return + if path == '/api/v1/users' and self.command == 'POST': + self._register(request_id, self._read_json()) + return + if path == '/api/v1/auth/sessions' and self.command == 'POST': + self._create_session(request_id) + return + if path.startswith('/api/v1/'): + user = self._authenticated_user(request_id) + if user is None: + return + user_id = user['id'] + if path == '/api/v1/auth/sessions/current' and self.command == 'DELETE': + self._logout(request_id, user) + return + if path == '/api/v1/users/me' and self.command == 'GET': + self._send(200, request_id, { + k: user[k] for k in ('id', 'username', 'display_name', 'created_at') + }) + return + if path == '/api/v1/chats' and self.command == 'POST': + body = self._read_json() + if set(body) != {'title', 'member_ids'}: + raise ApiError(400, 'invalid_request', 'Invalid chat fields') + title = required_text(body['title'], 'title', 100) + member_ids = body['member_ids'] + if (not isinstance(member_ids, list) or not member_ids or len(member_ids) > 100 + or any(not isinstance(value, str) for value in member_ids)): + raise ApiError(400, 'invalid_request', 'Invalid chat members') + normalized = [parse_uuid(value) for value in member_ids] + if len(set(normalized)) != len(normalized) or user_id not in normalized: + raise ApiError(400, 'invalid_request', 'Invalid chat members') + chat = self.server.store.create_chat( + user_id, title, normalized, now_iso() + ) + self._send(201, request_id, chat) + return + if path == '/api/v1/chats' and self.command == 'GET': + limit, cursor = self._query(route.query) + key = cursor_decode(cursor, 'chats') if cursor else None + items, next_key = self.server.store.list_chats(user_id, limit, key) + next_cursor = cursor_encode('chats', items[-1]) if next_key else None + self._send(200, request_id, {'items': items, 'next_cursor': next_cursor}) + return + match = re.fullmatch(r'/api/v1/chats/([^/]+)', path) + if match and self.command == 'GET': + chat_id = parse_uuid(match.group(1)) + self._send(200, request_id, self.server.store.get_chat(user_id, chat_id)) + return + match = re.fullmatch(r'/api/v1/chats/([^/]+)/messages', path) + if match: + chat_id = parse_uuid(match.group(1)) + resource = 'messages:' + chat_id + if self.command == 'POST': + body = self._read_json() + if not isinstance(body, dict) or set(body) != {'text'}: + raise ApiError(400, 'invalid_request', 'Invalid message fields') + text = body['text'] + if not isinstance(text, str) or len(text) > 4000 or not text.strip(): + raise ApiError(400, 'invalid_request', 'Message text must not be empty') + try: + text.encode('utf-8') + except UnicodeEncodeError: + raise ApiError(400, 'invalid_request', 'Invalid message text') from None + message = self.server.store.create_message( + user_id, chat_id, text, now_iso() + ) + self._send(201, request_id, message) + return + if self.command == 'GET': + limit, cursor = self._query(route.query) + key = cursor_decode(cursor, resource) if cursor else None + items, next_key = self.server.store.list_messages(user_id, chat_id, limit, key) + next_cursor = cursor_encode(resource, items[-1]) if next_key else None + self._send(200, request_id, {'items': items, 'next_cursor': next_cursor}) + return + raise ApiError(404, 'not_found', 'Resource not found') + except ApiError as error: + self._error(request_id, error) + except (BrokenPipeError, ConnectionResetError): + LOG.info('request disconnected request_id=%s', request_id) + except Exception: + LOG.exception('request failed request_id=%s', request_id) + self._error(request_id, ApiError(500, 'internal_error', 'Internal server error')) + + def do_GET(self): + self._dispatch() + + def do_POST(self): + self._dispatch() + + def do_PUT(self): + self._dispatch() + + def do_PATCH(self): + self._dispatch() + + def do_DELETE(self): + self._dispatch() + + def do_OPTIONS(self): + self._dispatch() + + +class MessengerServer(ThreadingHTTPServer): + daemon_threads = True + block_on_close = False + + def __init__(self, address, store, sessions): + super().__init__(address, MessengerHandler) + self.store = store + self.sessions = sessions + self.stopping = False + try: + spec = json.loads(OPENAPI_PATH.read_text(encoding='utf-8')) + spec['servers'] = [{'url': '/', 'description': 'Текущий адрес API'}] + self.openapi_json = json.dumps( + spec, ensure_ascii=False, separators=(',', ':') + ).encode('utf-8') + except (OSError, ValueError) as error: + self.server_close() + raise RuntimeError('Unable to load the OpenAPI contract') from error + + +def main(): + host = os.environ.get('APP_HOST', '0.0.0.0') + try: + port = int(os.environ.get('APP_PORT', '8080')) + except ValueError: + raise SystemExit('APP_PORT must be an integer') + if not 1 <= port <= 65535: + raise SystemExit('APP_PORT must be between 1 and 65535') + if os.environ.get('SESSION_STORE', 'redis').lower() != 'redis': + raise SystemExit('SESSION_STORE must be redis for this implementation') + + logging.basicConfig( + level=os.environ.get('LOG_LEVEL', 'INFO').upper(), + format='%(asctime)s %(levelname)s %(name)s %(message)s', + stream=sys.stdout, + ) + store = Store.connect() + sessions = SessionStore.from_env() + server = MessengerServer((host, port), store, sessions) + + def stop(_signum, _frame): + server.stopping = True + threading.Thread(target=server.shutdown, name='http-shutdown', daemon=True).start() + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + LOG.info('server started host=%s port=%d', host, port) + try: + server.serve_forever(poll_interval=0.2) + finally: + server.server_close() + store.close() + LOG.info('server stopped') + + +if __name__ == '__main__': + main() diff --git a/auth.py b/auth.py new file mode 100644 index 0000000..df4bee9 --- /dev/null +++ b/auth.py @@ -0,0 +1,77 @@ +import base64 +import binascii +import os +import secrets +from datetime import datetime, timedelta, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError + +from sessions import SessionStore + +PASSWORD_MIN = 12 +PASSWORD_MAX = 128 +_hasher = PasswordHasher() + + +def now_iso(): + return datetime.now(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def validate_password(password): + if not isinstance(password, str): + return False + length = len(password) + return PASSWORD_MIN <= length <= PASSWORD_MAX + + +def hash_password(password): + return _hasher.hash(password) + + +def verify_password(password_hash, password): + try: + return _hasher.verify(password_hash, password) + except VerifyMismatchError: + return False + + +def parse_basic_auth(header_value): + if not header_value or not header_value.startswith('Basic '): + return None + encoded = header_value[6:].strip() + try: + decoded = base64.b64decode(encoded, validate=True).decode('utf-8') + except (ValueError, UnicodeError, binascii.Error): + return None + if ':' not in decoded: + return None + username, password = decoded.split(':', 1) + if not username: + return None + return username, password + + +def parse_bearer_token(header_value): + if not header_value or not header_value.startswith('Bearer '): + return None + token = header_value[7:].strip() + return token or None + + +def session_ttl_seconds(): + try: + ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600')) + except ValueError: + ttl = 3600 + return max(1, ttl) + + +def create_session(store: SessionStore, user_id): + token = secrets.token_urlsafe(32) + ttl = session_ttl_seconds() + expires_at = ( + datetime.now(timezone.utc) + timedelta(seconds=ttl) + ).isoformat(timespec='microseconds').replace('+00:00', 'Z') + store.create(token, user_id, ttl) + return token, expires_at diff --git a/compose.track2-restore.yaml b/compose.track2-restore.yaml new file mode 100644 index 0000000..62db9b9 --- /dev/null +++ b/compose.track2-restore.yaml @@ -0,0 +1,19 @@ +services: + postgres_restore: + image: postgres:16.8-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: messenger_dev_only + POSTGRES_DB: postgres + ports: + - '5433:5432' + volumes: + - postgres_restore_data:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U messenger'] + interval: 3s + timeout: 3s + retries: 20 + +volumes: + postgres_restore_data: diff --git a/compose.ttl.yaml b/compose.ttl.yaml new file mode 100644 index 0000000..78a825b --- /dev/null +++ b/compose.ttl.yaml @@ -0,0 +1,5 @@ +# Overlay for session expiry checks: docker compose -f compose.yaml -f compose.ttl.yaml up -d --build +services: + api: + environment: + SESSION_TTL_SECONDS: 2 diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..fb14231 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,48 @@ +services: + postgres: + image: postgres:16.8-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: messenger_dev_only + POSTGRES_DB: messenger + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U messenger -d messenger'] + interval: 5s + timeout: 3s + retries: 10 + ports: + - '5432:5432' + + redis: + image: redis:7.4.2-alpine + command: ['redis-server', '--appendonly', 'yes', '--appendfsync', 'everysec'] + volumes: + - redis_data:/data + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + interval: 5s + timeout: 3s + retries: 10 + + api: + build: . + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + SESSION_STORE: redis + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DATABASE_URL: postgresql://messenger:messenger_dev_only@postgres:5432/messenger + CACHE_URL: redis://redis:6379/0 + ports: + - '${APP_PORT:-8080}:8080' + +volumes: + postgres_data: + redis_data: diff --git a/db.py b/db.py new file mode 100644 index 0000000..0fe4ecb --- /dev/null +++ b/db.py @@ -0,0 +1,256 @@ +import os +import uuid +from pathlib import Path + +from psycopg import errors +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +MIGRATIONS_DIR = Path(__file__).resolve().parent / 'migrations' + + +class ApiError(Exception): + def __init__(self, status, code, message): + super().__init__(message) + self.status = status + self.code = code + self.message = message + + +def read_database_url(): + path = os.environ.get('DATABASE_URL_FILE') + if path and os.path.isfile(path): + return Path(path).read_text(encoding='utf-8').strip() + url = os.environ.get('DATABASE_URL', '').strip() + if not url: + raise RuntimeError('DATABASE_URL or DATABASE_URL_FILE is required') + return url + + +class Store: + def __init__(self, pool: ConnectionPool): + self.pool = pool + + @classmethod + def connect(cls): + pool = ConnectionPool( + conninfo=read_database_url(), + min_size=1, + max_size=10, + timeout=5.0, + kwargs={'row_factory': dict_row}, + open=True, + ) + store = cls(pool) + store.run_migrations() + return store + + def close(self): + self.pool.close() + + def ping(self): + with self.pool.connection() as conn: + conn.execute('SELECT 1') + + def run_migrations(self): + with self.pool.connection() as conn: + conn.execute( + 'CREATE TABLE IF NOT EXISTS schema_migrations ' + '(version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())' + ) + for path in sorted(MIGRATIONS_DIR.glob('*.sql')): + version = path.name + row = conn.execute( + 'SELECT 1 FROM schema_migrations WHERE version = %s', + (version,), + ).fetchone() + if row: + continue + sql = path.read_text(encoding='utf-8') + with conn.transaction(): + conn.execute(sql) + conn.execute( + 'INSERT INTO schema_migrations (version) VALUES (%s)', + (version,), + ) + + @staticmethod + def _user_row(row): + return { + 'id': str(row['id']), + 'username': row['username'], + 'display_name': row['display_name'], + 'created_at': row['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'), + } + + def register_user(self, username, display_name, password_hash, created_at): + user_id = str(uuid.uuid4()) + try: + with self.pool.connection() as conn: + conn.execute( + 'INSERT INTO users (id, username, display_name, password_hash, created_at) ' + 'VALUES (%s, %s, %s, %s, %s)', + (user_id, username, display_name, password_hash, created_at), + ) + except errors.UniqueViolation: + raise ApiError(409, 'conflict', 'Username already exists') from None + return { + 'id': user_id, + 'username': username, + 'display_name': display_name, + 'created_at': created_at, + } + + def get_user(self, user_id): + with self.pool.connection() as conn: + row = conn.execute( + 'SELECT id, username, display_name, created_at FROM users WHERE id = %s', + (user_id,), + ).fetchone() + if row is None: + return None + return self._user_row(row) + + def get_user_with_password(self, username): + with self.pool.connection() as conn: + row = conn.execute( + 'SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s', + (username,), + ).fetchone() + return row + + def create_chat(self, user_id, title, member_ids, created_at): + chat_id = str(uuid.uuid4()) + try: + with self.pool.connection() as conn: + with conn.transaction(): + rows = conn.execute( + 'SELECT id FROM users WHERE id = ANY(%s)', + (member_ids,), + ).fetchall() + if len(rows) != len(set(member_ids)): + raise ApiError(400, 'invalid_request', 'Unknown chat member') + conn.execute( + 'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)', + (chat_id, title, created_at), + ) + for index, member_id in enumerate(member_ids): + conn.execute( + 'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)', + (chat_id, member_id, index), + ) + except ApiError: + raise + except errors.Error: + raise ApiError(500, 'internal_error', 'Internal server error') from None + return { + 'id': chat_id, + 'title': title, + 'member_ids': list(member_ids), + 'created_at': created_at, + } + + def _is_member(self, conn, chat_id, user_id): + row = conn.execute( + 'SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s', + (chat_id, user_id), + ).fetchone() + return row is not None + + def get_chat(self, user_id, chat_id): + with self.pool.connection() as conn: + if not self._is_member(conn, chat_id, user_id): + raise ApiError(404, 'not_found', 'Chat not found') + chat = conn.execute( + 'SELECT id, title, created_at FROM chats WHERE id = %s', + (chat_id,), + ).fetchone() + members = conn.execute( + 'SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY position', + (chat_id,), + ).fetchall() + return { + 'id': str(chat['id']), + 'title': chat['title'], + 'member_ids': [str(row['user_id']) for row in members], + 'created_at': chat['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'), + } + + def list_chats(self, user_id, limit, cursor_key): + query = ( + 'SELECT c.id, c.title, c.created_at ' + 'FROM chats c ' + 'JOIN chat_members cm ON cm.chat_id = c.id AND cm.user_id = %s ' + ) + params = [user_id] + if cursor_key: + query += 'WHERE (c.created_at, c.id) > (%s, %s) ' + params.extend(cursor_key) + query += 'ORDER BY c.created_at ASC, c.id ASC LIMIT %s' + params.append(limit + 1) + with self.pool.connection() as conn: + rows = conn.execute(query, params).fetchall() + has_more = len(rows) > limit + page = rows[:limit] + items = [] + for row in page: + chat_id = str(row['id']) + items.append(self.get_chat(user_id, chat_id)) + next_cursor = None + if has_more: + last = page[-1] + next_cursor = (last['created_at'], str(last['id'])) + return items, next_cursor + + def create_message(self, user_id, chat_id, text, created_at): + message_id = str(uuid.uuid4()) + with self.pool.connection() as conn: + with conn.transaction(): + if not self._is_member(conn, chat_id, user_id): + raise ApiError(404, 'not_found', 'Chat not found') + conn.execute( + 'INSERT INTO messages (id, chat_id, sender_id, text, created_at) ' + 'VALUES (%s, %s, %s, %s, %s)', + (message_id, chat_id, user_id, text, created_at), + ) + return { + 'id': message_id, + 'chat_id': chat_id, + 'sender_id': user_id, + 'text': text, + 'created_at': created_at, + } + + def list_messages(self, user_id, chat_id, limit, cursor_key): + with self.pool.connection() as conn: + if not self._is_member(conn, chat_id, user_id): + raise ApiError(404, 'not_found', 'Chat not found') + query = ( + 'SELECT id, chat_id, sender_id, text, created_at FROM messages ' + 'WHERE chat_id = %s ' + ) + params = [chat_id] + if cursor_key: + query += 'AND (created_at, id) > (%s, %s) ' + params.extend(cursor_key) + query += 'ORDER BY created_at ASC, id ASC LIMIT %s' + params.append(limit + 1) + with self.pool.connection() as conn: + rows = conn.execute(query, params).fetchall() + has_more = len(rows) > limit + page = rows[:limit] + items = [ + { + 'id': str(row['id']), + 'chat_id': str(row['chat_id']), + 'sender_id': str(row['sender_id']), + 'text': row['text'], + 'created_at': row['created_at'].isoformat(timespec='microseconds').replace('+00:00', 'Z'), + } + for row in page + ] + next_cursor = None + if has_more: + last = page[-1] + next_cursor = (last['created_at'], str(last['id'])) + return items, next_cursor diff --git a/evidence/explain.txt b/evidence/explain.txt new file mode 100644 index 0000000..3d9d594 --- /dev/null +++ b/evidence/explain.txt @@ -0,0 +1,13 @@ +EXPLAIN (ANALYZE, BUFFERS) on 10_000 messages in one chat. + +Command (from repo root, stack running): + docker compose run --rm \ + -e DATABASE_URL=postgresql://messenger:messenger_dev_only@postgres:5432/messenger \ + -e EXPLAIN_MESSAGE_COUNT=10000 \ + -v ./scripts:/scripts:ro \ + api python /scripts/explain_messages.py + +Observed plan uses index messages_chat_history_idx (Bitmap Index Scan on chat_id), +not Seq Scan on all messages. Execution Time ~2.3 ms on local Docker. + +Keyset pagination in API: WHERE chat_id = $1 AND (created_at, id) > ($cursor...) LIMIT n+1 diff --git a/evidence/track2/README.md b/evidence/track2/README.md new file mode 100644 index 0000000..9311b22 --- /dev/null +++ b/evidence/track2/README.md @@ -0,0 +1,13 @@ +# Track 2 — план запросов и восстановление + +Гипотеза: для skewed-нагрузки (70k сообщений в одном чате) keyset по `(chat_id, created_at, id)` даёт стабильное чтение; большой `OFFSET` и лишний индекс `(chat_id)` ухудшают latency/запись. + +Повтор: + +```sh +py -3 scripts/track2_run.py +py -3 scripts/track2_backup_restore.py +RUN_TRACK2_VERIFY=1 py -3 -m unittest student-tests.test_track2 +``` + +Артефакты: `benchmark.json`, `backup.dump`, `restore_report.json`. diff --git a/evidence/track2/backup.dump b/evidence/track2/backup.dump new file mode 100644 index 0000000..d306d83 Binary files /dev/null and b/evidence/track2/backup.dump differ diff --git a/evidence/track2/benchmark.json b/evidence/track2/benchmark.json new file mode 100644 index 0000000..0baf8cc --- /dev/null +++ b/evidence/track2/benchmark.json @@ -0,0 +1,28 @@ +{ + "hypothesis": "Keyset pagination on (chat_id, created_at, id) avoids large offsets; redundant (chat_id)-only index does not improve reads but slows writes.", + "hot_chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df", + "read_plans": { + "keyset_limit_50": { + "node_type": "Index Only Scan", + "index_name": "messages_chat_history_idx", + "execution_ms": 0.349, + "plan_rows": 11 + }, + "offset_50000_limit_50": { + "node_type": "Index Only Scan", + "index_name": "messages_chat_history_idx", + "execution_ms": 7.043, + "plan_rows": 1 + }, + "keyset_with_extra_chat_id_index": { + "node_type": "Index Scan", + "index_name": "messages_chat_id_only_idx", + "execution_ms": 9.797, + "plan_rows": 11 + } + }, + "write_messages_per_second": { + "with_extra_chat_id_index": 18199.5, + "production_indexes_only": 20166.9 + } +} \ No newline at end of file diff --git a/evidence/track2/restore_report.json b/evidence/track2/restore_report.json new file mode 100644 index 0000000..eb6382f --- /dev/null +++ b/evidence/track2/restore_report.json @@ -0,0 +1,69 @@ +{ + "source": { + "messages": 111012, + "users": 43, + "sample": [ + { + "id": "d4bff878-42c2-4ed2-8261-e2fc42583f65", + "chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df", + "text": "w" + }, + { + "id": "c561eb24-1381-491f-b78c-b9fe460015b5", + "chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df", + "text": "w" + }, + { + "id": "e0789e8a-6c7b-40e9-8141-64d8547de768", + "chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df", + "text": "w" + }, + { + "id": "d0e5a399-932d-4006-91b1-f1cc26501af5", + "chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df", + "text": "w" + }, + { + "id": "de8288c2-a7fb-496b-8be7-cbfe82d138c5", + "chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df", + "text": "w" + } + ] + }, + "restored": { + "messages": 111012, + "users": 43, + "sample": [ + { + "id": "d4bff878-42c2-4ed2-8261-e2fc42583f65", + "chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df", + "text": "w" + }, + { + "id": "c561eb24-1381-491f-b78c-b9fe460015b5", + "chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df", + "text": "w" + }, + { + "id": "e0789e8a-6c7b-40e9-8141-64d8547de768", + "chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df", + "text": "w" + }, + { + "id": "d0e5a399-932d-4006-91b1-f1cc26501af5", + "chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df", + "text": "w" + }, + { + "id": "de8288c2-a7fb-496b-8be7-cbfe82d138c5", + "chat_id": "01aeb3d4-8235-4964-8981-ace0cec560df", + "text": "w" + } + ] + }, + "counts_match": true, + "sample_match": true, + "restore_seconds": 7.09, + "dump_bytes": 2899617, + "data_loss_boundary": "Logical pg_dump is consistent at dump start; concurrent writes during dump are not included. WAL/PITR not used — RPO equals changes after dump began." +} \ No newline at end of file diff --git a/migrations/001_schema.sql b/migrations/001_schema.sql new file mode 100644 index 0000000..a618751 --- /dev/null +++ b/migrations/001_schema.sql @@ -0,0 +1,34 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS users_username_key ON users (username); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE, + position INT NOT NULL, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX IF NOT EXISTS chat_members_user_id_idx ON chat_members (user_id); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE, + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS messages_chat_history_idx ON messages (chat_id, created_at ASC, id ASC); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..bb6ebc7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +argon2-cffi==23.1.0 +psycopg[binary,pool]==3.2.9 +redis==5.2.1 diff --git a/scripts/explain_messages.py b/scripts/explain_messages.py new file mode 100644 index 0000000..d00a34b --- /dev/null +++ b/scripts/explain_messages.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Seed >=10k messages and print EXPLAIN (ANALYZE, BUFFERS) for history pagination.""" +import os +import sys +import uuid +from datetime import datetime, timezone + +import psycopg + + +def main(): + url = os.environ.get('DATABASE_URL') + if not url: + print('Set DATABASE_URL', file=sys.stderr) + raise SystemExit(2) + chat_id = os.environ.get('EXPLAIN_CHAT_ID') + user_id = os.environ.get('EXPLAIN_USER_ID') + count = int(os.environ.get('EXPLAIN_MESSAGE_COUNT', '10000')) + with psycopg.connect(url) as conn: + if not chat_id or not user_id: + user_id = str(uuid.uuid4()) + chat_id = str(uuid.uuid4()) + now = datetime.now(timezone.utc) + conn.execute( + 'INSERT INTO users (id, username, display_name, password_hash, created_at) ' + 'VALUES (%s, %s, %s, %s, %s)', + (user_id, 'bench_' + uuid.uuid4().hex[:8], 'Bench', 'x', now), + ) + conn.execute( + 'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)', + (chat_id, 'Bench', now), + ) + conn.execute( + 'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, 0)', + (chat_id, user_id), + ) + print(f'created chat_id={chat_id} user_id={user_id}') + existing = conn.execute( + 'SELECT count(*) FROM messages WHERE chat_id = %s', + (chat_id,), + ).fetchone()[0] + need = max(0, count - existing) + base = datetime.now(timezone.utc) + with conn.cursor() as cur: + for index in range(need): + cur.execute( + 'INSERT INTO messages (id, chat_id, sender_id, text, created_at) ' + 'VALUES (%s, %s, %s, %s, %s)', + ( + str(uuid.uuid4()), + chat_id, + user_id, + 'm', + base, + ), + ) + conn.commit() + plan = conn.execute( + 'EXPLAIN (ANALYZE, BUFFERS) ' + 'SELECT id FROM messages WHERE chat_id = %s ' + 'ORDER BY created_at ASC, id ASC LIMIT 50', + (chat_id,), + ).fetchall() + for row in plan: + print(row[0]) + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.bat b/scripts/restart.bat new file mode 100644 index 0000000..e688fa9 --- /dev/null +++ b/scripts/restart.bat @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +docker compose restart api postgres redis diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 0000000..243de43 --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +"""Cross-platform restart hook; on Windows use: ACTION_SCRIPT=scripts/restart.py""" +import subprocess +import sys +from pathlib import Path + +root = Path(__file__).resolve().parents[1] +result = subprocess.run( + ['docker', 'compose', 'restart', 'api', 'postgres', 'redis'], + cwd=root, + check=False, +) +raise SystemExit(result.returncode) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..b8c44ae --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +root="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$root" +docker compose restart api postgres redis diff --git a/scripts/track2_backup_restore.py b/scripts/track2_backup_restore.py new file mode 100644 index 0000000..66687ff --- /dev/null +++ b/scripts/track2_backup_restore.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""pg_dump → fresh Postgres volume → verification via psql (no local psycopg).""" +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / 'evidence' / 'track2' +DUMP = OUT / 'backup.dump' +MAIN = ['docker', 'compose', '-f', 'compose.yaml'] +RESTORE = ['docker', 'compose', '-f', 'compose.track2-restore.yaml'] +PSQL = os.environ.get('TRACK2_PSQL_SERVICE', 'postgres') +RESTORE_SVC = 'postgres_restore' + + +def run(cmd, **kwargs): + subprocess.run(cmd, cwd=ROOT, check=True, **kwargs) + + +def psql(service, compose, database, sql): + out = subprocess.check_output( + compose + ['exec', '-T', service, 'psql', '-U', 'messenger', '-d', database, '-tAc', sql], + cwd=ROOT, + text=True, + ) + return out.strip() + + +def snapshot(service, compose, database): + messages = int(psql(service, compose, database, 'SELECT count(*) FROM messages')) + users = int(psql(service, compose, database, 'SELECT count(*) FROM users')) + raw = subprocess.check_output( + compose + [ + 'exec', '-T', service, 'psql', '-U', 'messenger', '-d', database, '-tAc', + "SELECT id::text || '|' || chat_id::text || '|' || text FROM messages " + 'ORDER BY created_at DESC LIMIT 5', + ], + cwd=ROOT, + text=True, + ).strip() + sample = [] + if raw: + for line in raw.splitlines(): + parts = line.split('|', 2) + sample.append({'id': parts[0], 'chat_id': parts[1], 'text': parts[2]}) + return {'messages': messages, 'users': users, 'sample': sample} + + +def main(): + OUT.mkdir(parents=True, exist_ok=True) + started = time.perf_counter() + run(RESTORE + ['up', '-d', '--wait']) + with DUMP.open('wb') as handle: + run(MAIN + ['exec', '-T', PSQL, 'pg_dump', '-U', 'messenger', '-Fc', 'messenger'], stdout=handle) + before = snapshot(PSQL, MAIN, 'messenger') + run( + RESTORE + ['exec', '-T', RESTORE_SVC, 'psql', '-U', 'messenger', '-d', 'postgres', '-c', + 'DROP DATABASE IF EXISTS messenger_restore'], + ) + run( + RESTORE + ['exec', '-T', RESTORE_SVC, 'psql', '-U', 'messenger', '-d', 'postgres', '-c', + 'CREATE DATABASE messenger_restore'], + ) + run(RESTORE + ['cp', str(DUMP), f'{RESTORE_SVC}:/tmp/backup.dump']) + restore = subprocess.run( + RESTORE + [ + 'exec', '-T', RESTORE_SVC, + 'pg_restore', '-U', 'messenger', '-d', 'messenger_restore', + '--no-owner', '--no-privileges', '/tmp/backup.dump', + ], + cwd=ROOT, + ) + if restore.returncode not in (0, 1): + restore.check_returncode() + after = snapshot(RESTORE_SVC, RESTORE, 'messenger_restore') + elapsed = time.perf_counter() - started + ok = ( + before['messages'] == after['messages'] + and before['users'] == after['users'] + and before['sample'] == after['sample'] + ) + report = { + 'source': before, + 'restored': after, + 'counts_match': before['messages'] == after['messages'], + 'sample_match': before['sample'] == after['sample'], + 'restore_seconds': round(elapsed, 2), + 'dump_bytes': DUMP.stat().st_size, + 'data_loss_boundary': ( + 'Logical pg_dump is consistent at dump start; concurrent writes during dump are not ' + 'included. WAL/PITR not used — RPO equals changes after dump began.' + ), + } + (OUT / 'restore_report.json').write_text( + json.dumps(report, indent=2, ensure_ascii=False), + encoding='utf-8', + ) + print(json.dumps(report, indent=2, ensure_ascii=False)) + if not ok: + raise SystemExit(1) + print('PASS: restore verification') + + +if __name__ == '__main__': + main() diff --git a/scripts/track2_benchmark.py b/scripts/track2_benchmark.py new file mode 100644 index 0000000..5bcf422 --- /dev/null +++ b/scripts/track2_benchmark.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Compare index plans and INSERT cost; write evidence/track2/benchmark.json.""" +import json +import os +import sys +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path + +import psycopg + +ROOT = Path(__file__).resolve().parents[1] +OUT = Path(os.environ.get('TRACK2_EVIDENCE_DIR', str(ROOT / 'evidence' / 'track2'))) +DATABASE_URL = os.environ.get('DATABASE_URL', '') +HOT_CHAT_TITLE = os.environ.get('TRACK2_HOT_CHAT_TITLE', 'track2_hot') + + +def _find_index(plan): + if plan.get('Index Name'): + return plan.get('Index Name'), plan.get('Node Type') + for child in plan.get('Plans') or []: + found = _find_index(child) + if found: + return found + return None, plan.get('Node Type') + + +def explain(conn, sql, params): + rows = conn.execute('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ' + sql, params).fetchone()[0] + plan = rows[0]['Plan'] + index_name, node_type = _find_index(plan) + return { + 'node_type': node_type, + 'index_name': index_name, + 'execution_ms': rows[0]['Execution Time'], + 'plan_rows': plan.get('Plan Rows'), + } + + +def insert_batch(conn, chat_id, user_id, n): + base = datetime.now(timezone.utc) + start = time.perf_counter() + with conn.cursor() as cur: + cur.executemany( + 'INSERT INTO messages (id, chat_id, sender_id, text, created_at) ' + 'VALUES (%s, %s, %s, %s, %s)', + [ + (str(uuid.uuid4()), chat_id, user_id, 'w', base) + for _ in range(n) + ], + ) + conn.commit() + elapsed = time.perf_counter() - start + return n / elapsed + + +def main(): + if not DATABASE_URL: + print('DATABASE_URL required', file=sys.stderr) + raise SystemExit(2) + OUT.mkdir(parents=True, exist_ok=True) + with psycopg.connect(DATABASE_URL) as conn: + hot = conn.execute( + 'SELECT c.id FROM chats c WHERE c.title = %s', + (HOT_CHAT_TITLE,), + ).fetchone() + if not hot: + print('Run track2_seed.py first', file=sys.stderr) + raise SystemExit(2) + hot_id = str(hot[0]) + user = conn.execute( + "SELECT id FROM users WHERE username = 'track2_bench'" + ).fetchone() + user_id = str(user[0]) + keyset_sql = ( + 'SELECT id FROM messages WHERE chat_id = %s ' + 'ORDER BY created_at ASC, id ASC LIMIT 50' + ) + offset_sql = ( + 'SELECT id FROM messages WHERE chat_id = %s ' + 'ORDER BY created_at ASC, id ASC OFFSET 50000 LIMIT 50' + ) + plan_keyset = explain(conn, keyset_sql, (hot_id,)) + plan_offset = explain(conn, offset_sql, (hot_id,)) + conn.execute('DROP INDEX IF EXISTS messages_chat_id_only_idx') + conn.execute('CREATE INDEX messages_chat_id_only_idx ON messages (chat_id)') + conn.commit() + plan_keyset_with_extra = explain(conn, keyset_sql, (hot_id,)) + write_before = insert_batch(conn, hot_id, user_id, 500) + conn.execute('DROP INDEX IF EXISTS messages_chat_id_only_idx') + conn.commit() + write_after = insert_batch(conn, hot_id, user_id, 500) + result = { + 'hypothesis': ( + 'Keyset pagination on (chat_id, created_at, id) avoids large offsets; ' + 'redundant (chat_id)-only index does not improve reads but slows writes.' + ), + 'hot_chat_id': hot_id, + 'read_plans': { + 'keyset_limit_50': plan_keyset, + 'offset_50000_limit_50': plan_offset, + 'keyset_with_extra_chat_id_index': plan_keyset_with_extra, + }, + 'write_messages_per_second': { + 'with_extra_chat_id_index': round(write_before, 1), + 'production_indexes_only': round(write_after, 1), + }, + } + out_path = OUT / 'benchmark.json' + out_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding='utf-8') + print(json.dumps(result, indent=2, ensure_ascii=False)) + print('wrote', out_path) + + +if __name__ == '__main__': + main() diff --git a/scripts/track2_run.py b/scripts/track2_run.py new file mode 100644 index 0000000..1d0eeea --- /dev/null +++ b/scripts/track2_run.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Run full track 2 pipeline (seed, benchmark, backup/restore).""" +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DB = 'postgresql://messenger:messenger_dev_only@postgres:5432/messenger' +EVIDENCE = ROOT / 'evidence' / 'track2' +EVIDENCE.mkdir(parents=True, exist_ok=True) + + +def docker_python(script): + subprocess.run( + [ + 'docker', 'compose', 'run', '--rm', + '-e', 'DATABASE_URL=' + DB, + '-e', 'TRACK2_EVIDENCE_DIR=/evidence', + '-v', f'{ROOT / "scripts"}:/scripts:ro', + '-v', f'{EVIDENCE}:/evidence', + 'api', 'python', f'/scripts/{script}', + ], + cwd=ROOT, + check=True, + ) + + +def main(): + for name in ('track2_seed.py', 'track2_benchmark.py'): + print('===', name, '===') + docker_python(name) + print('=== track2_backup_restore.py ===') + subprocess.run([sys.executable, str(ROOT / 'scripts' / 'track2_backup_restore.py')], cwd=ROOT, check=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/track2_seed.py b/scripts/track2_seed.py new file mode 100644 index 0000000..a9b9697 --- /dev/null +++ b/scripts/track2_seed.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Seed >=100k messages with skewed chat sizes (track 2).""" +import os +import sys +import uuid +from datetime import datetime, timedelta, timezone + +import psycopg + +TARGET = int(os.environ.get('TRACK2_MESSAGE_COUNT', '100000')) +DATABASE_URL = os.environ.get('DATABASE_URL', '') + + +def main(): + if not DATABASE_URL: + print('DATABASE_URL required', file=sys.stderr) + raise SystemExit(2) + base = datetime.now(timezone.utc) + with psycopg.connect(DATABASE_URL) as conn: + hot_exists = conn.execute( + "SELECT 1 FROM chats WHERE title = 'track2_hot'" + ).fetchone() + if hot_exists: + total = conn.execute('SELECT count(*) FROM messages').fetchone()[0] + print(f'skip_seed existing track2 dataset messages_total={total}') + return + row = conn.execute( + "SELECT id FROM users WHERE username = 'track2_bench'" + ).fetchone() + if row: + user_id = str(row[0]) + else: + user_id = str(uuid.uuid4()) + conn.execute( + 'INSERT INTO users (id, username, display_name, password_hash, created_at) ' + 'VALUES (%s, %s, %s, %s, %s)', + (user_id, 'track2_bench', 'Track2', 'x', base), + ) + plans = [ + ('hot', int(TARGET * 0.70)), + ('medium_a', int(TARGET * 0.05)), + ('medium_b', int(TARGET * 0.05)), + ('medium_c', int(TARGET * 0.05)), + ('medium_d', int(TARGET * 0.05)), + ('medium_e', int(TARGET * 0.05)), + ] + tail = TARGET - sum(count for _, count in plans) + small_chats = 50 + per_small = max(1, tail // small_chats) + chat_ids = [] + for name, count in plans: + chat_id = str(uuid.uuid4()) + chat_ids.append((chat_id, count, name)) + conn.execute( + 'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)', + (chat_id, 'track2_' + name, base), + ) + conn.execute( + 'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, 0)', + (chat_id, user_id), + ) + for index in range(small_chats): + chat_id = str(uuid.uuid4()) + chat_ids.append((chat_id, per_small, f'small_{index}')) + conn.execute( + 'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)', + (chat_id, 'track2_small_' + str(index), base), + ) + conn.execute( + 'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, 0)', + (chat_id, user_id), + ) + total = 0 + with conn.cursor() as cur: + for chat_id, count, _name in chat_ids: + rows = [] + for offset in range(count): + rows.append( + ( + str(uuid.uuid4()), + chat_id, + user_id, + 't', + base + timedelta(microseconds=offset), + ) + ) + cur.executemany( + 'INSERT INTO messages (id, chat_id, sender_id, text, created_at) ' + 'VALUES (%s, %s, %s, %s, %s)', + rows, + ) + total += count + conn.commit() + actual = conn.execute('SELECT count(*) FROM messages').fetchone()[0] + hot = conn.execute( + "SELECT count(*) FROM messages m JOIN chats c ON c.id = m.chat_id WHERE c.title = 'track2_hot'" + ).fetchone()[0] + print(f'seeded_messages={total} table_total={actual} hot_chat_messages={hot}') + print(f'benchmark_user_id={user_id}') + for chat_id, count, name in chat_ids[:3]: + print(f'sample_chat {name} id={chat_id} messages={count}') + + +if __name__ == '__main__': + main() diff --git a/sessions.py b/sessions.py new file mode 100644 index 0000000..e579fb0 --- /dev/null +++ b/sessions.py @@ -0,0 +1,43 @@ +import os + +import redis + + +class SessionStore: + def __init__(self, url, connect_timeout=2.0, socket_timeout=2.0): + self._url = url + self._client = redis.from_url( + url, + decode_responses=True, + socket_connect_timeout=connect_timeout, + socket_timeout=socket_timeout, + health_check_interval=30, + ) + self._prefix = 'sess:' + + @classmethod + def from_env(cls): + url = _read_config('CACHE_URL_FILE', 'CACHE_URL') + if not url: + raise RuntimeError('CACHE_URL or CACHE_URL_FILE is required when SESSION_STORE=redis') + return cls(url) + + def ping(self): + return self._client.ping() + + def create(self, token, user_id, ttl_seconds): + self._client.setex(self._prefix + token, ttl_seconds, user_id) + + def get_user_id(self, token): + return self._client.get(self._prefix + token) + + def revoke(self, token): + return self._client.delete(self._prefix + token) > 0 + + +def _read_config(file_env, value_env): + path = os.environ.get(file_env) + if path and os.path.isfile(path): + with open(path, encoding='utf-8') as handle: + return handle.read().strip() + return os.environ.get(value_env, '').strip() diff --git a/student-tests/test_dependencies.py b/student-tests/test_dependencies.py new file mode 100644 index 0000000..752face --- /dev/null +++ b/student-tests/test_dependencies.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Проверка отказа Redis/PostgreSQL (уровень 4). Требует Docker и RUN_INFRA_TEST=1.""" +import json +import os +import subprocess +import sys +import time +import unittest +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/') +def compose_argv(): + override = os.environ.get('COMPOSE', '').strip() + if override: + return override.split() + return ['docker', 'compose'] + + +def compose(*args): + result = subprocess.run( + [*compose_argv(), *args], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + COMPOSE + ' ' + ' '.join(args) + ' failed: ' + (result.stderr or result.stdout) + ) + + +def http(method, path, token=None, expected=None): + headers = {'Accept': 'application/json'} + if token: + headers['Authorization'] = 'Bearer ' + token + req = urllib.request.Request(BASE + path, headers=headers, method=method) + try: + response = urllib.request.urlopen(req, timeout=5) + except urllib.error.HTTPError as error: + response = error + with response: + body = response.read() + payload = json.loads(body) if body else None + if expected is not None: + if isinstance(expected, int): + expected = [expected] + if response.code not in expected: + raise AssertionError( + f'{method} {path}: expected {expected}, got {response.code} {payload}' + ) + return response.code, payload + + +def wait_ready(deadline=60): + end = time.monotonic() + deadline + while time.monotonic() < end: + try: + code, _ = http('GET', '/health/ready') + if code == 200: + return + except (urllib.error.URLError, TimeoutError, ConnectionError, json.JSONDecodeError): + pass + time.sleep(0.5) + raise TimeoutError('API did not become ready') + + +@unittest.skipUnless( + os.environ.get('RUN_INFRA_TEST') == '1', + 'Set RUN_INFRA_TEST=1 to run Docker dependency failure tests', +) +class DependencyFailureTests(unittest.TestCase): + token = None + + @classmethod + def setUpClass(cls): + wait_ready() + sys.path.insert(0, str(ROOT / 'tests')) + from client import Client # noqa: WPS433 + + actor = Client(BASE).user('dep') + cls.token = actor['token'] + + def tearDown(self): + compose('start', 'redis', 'postgres') + wait_ready() + + def test_redis_down_ready_and_auth(self): + compose('stop', 'redis') + time.sleep(1) + code, payload = http('GET', '/health/ready') + self.assertEqual(code, 503) + self.assertEqual(payload['error']['code'], 'unavailable') + code, payload = http('GET', '/api/v1/users/me', token=self.token) + self.assertEqual(code, 503) + self.assertEqual(payload['error']['code'], 'unavailable') + + def test_postgres_down_ready(self): + compose('stop', 'postgres') + time.sleep(1) + code, payload = http('GET', '/health/ready') + self.assertEqual(code, 503) + self.assertEqual(payload['error']['code'], 'unavailable') + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_lab2.py b/student-tests/test_lab2.py new file mode 100644 index 0000000..087e7f9 --- /dev/null +++ b/student-tests/test_lab2.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Дополнительные проверки уровня 4: auth, expiry, гонка username, rollback чата.""" +import base64 +import json +import os +import sys +import time +import unittest +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check # noqa: E402 + + +BASE = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/') + + +def raw_request(method, path, expected, body=None, headers=None): + data = None + h = {'Accept': 'application/json'} + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode('utf-8') + h['Content-Type'] = 'application/json' + h.update(headers or {}) + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + response = urllib.request.urlopen(req, timeout=10) + except urllib.error.HTTPError as error: + response = error + with response: + payload = response.read() + check(response.code == expected, f'{method} {path}: expected {expected}, got {response.code}') + return response.code, response.headers, json.loads(payload) if payload else None + + +class Lab2Grade4Tests(unittest.TestCase): + def test_logout_revokes_session(self): + c = Client(BASE) + actor = c.user('logout') + c.call('GET', '/api/v1/users/me', actor=actor, schema='User') + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor) + c.call('GET', '/api/v1/users/me', 401, actor=actor) + + def test_wrong_password_rejected(self): + c = Client(BASE) + actor = c.user('pwd') + bad = base64.b64encode( + (actor['registration']['username'] + ':wrong-password-x').encode() + ).decode() + c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': 'Basic ' + bad}) + + def test_readiness_requires_dependencies(self): + c = Client(BASE) + c.call('GET', '/health/ready', schema='Health') + + @unittest.skipUnless( + os.environ.get('RUN_EXPIRY_TEST') == '1', + 'Set RUN_EXPIRY_TEST=1 and SESSION_TTL_SECONDS=2 on the API, then rerun', + ) + def test_session_expiry(self): + c = Client(BASE) + username = 'u_exp_' + uuid.uuid4().hex[:12] + password = 'T3st-' + uuid.uuid4().hex + body = {'username': username, 'display_name': 'Expiry', 'password': password} + user = c.call('POST', '/api/v1/users', 201, body=body, schema='User')['json'] + credential = base64.b64encode((username + ':' + password).encode()).decode() + session = c.call( + 'POST', '/api/v1/auth/sessions', 201, + headers={'Authorization': 'Basic ' + credential}, + schema='Session', + )['json'] + actor = {'id': user['id'], 'user': user, 'registration': body, 'token': session['token']} + time.sleep(3) + c.call('GET', '/api/v1/users/me', 401, actor=actor) + + def test_duplicate_username_race(self): + c = Client(BASE) + username = 'u_race_' + uuid.uuid4().hex[:10] + password = 'T3st-' + uuid.uuid4().hex + body = { + 'username': username, + 'display_name': 'Race', + 'password': password, + } + + def register(): + data = json.dumps(body, ensure_ascii=False).encode('utf-8') + req = urllib.request.Request( + BASE + '/api/v1/users', + data=data, + headers={'Accept': 'application/json', 'Content-Type': 'application/json'}, + method='POST', + ) + try: + response = urllib.request.urlopen(req, timeout=10) + except urllib.error.HTTPError as error: + return error.code + with response: + return response.code + + with ThreadPoolExecutor(max_workers=8) as pool: + statuses = list(pool.map(lambda _: register(), range(8))) + check(all(code in (201, 409) for code in statuses), f'unexpected race statuses: {set(statuses)}') + check(sum(1 for code in statuses if code == 201) == 1, 'expected exactly one 201') + + def test_chat_creation_rollback_unknown_member(self): + c = Client(BASE) + alice = c.user('rb_alice') + bob = c.user('rb_bob') + fake = str(uuid.uuid4()) + c.call( + 'POST', '/api/v1/chats', 400, actor=alice, + body={'title': 'Bad', 'member_ids': [alice['id'], fake]}, + ) + listed = c.call('GET', '/api/v1/chats?limit=100', actor=alice, schema='ChatPage')['json'] + check(all(item['title'] != 'Bad' for item in listed['items']), 'chat row leaked after rollback') + listed_bob = c.call('GET', '/api/v1/chats?limit=100', actor=bob, schema='ChatPage')['json'] + check(all(item['title'] != 'Bad' for item in listed_bob['items']), 'chat row leaked to bob') + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_track2.py b/student-tests/test_track2.py new file mode 100644 index 0000000..40abdbf --- /dev/null +++ b/student-tests/test_track2.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Sanity checks for track 2 evidence artifacts.""" +import json +import os +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / 'evidence' / 'track2' + + +@unittest.skipUnless( + os.environ.get('RUN_TRACK2_VERIFY') == '1', + 'Set RUN_TRACK2_VERIFY=1 after running track2 scripts', +) +class Track2EvidenceTests(unittest.TestCase): + def test_benchmark_artifact(self): + path = EVIDENCE / 'benchmark.json' + self.assertTrue(path.is_file(), 'missing benchmark.json — run scripts/track2_benchmark.py') + data = json.loads(path.read_text(encoding='utf-8')) + keyset = data['read_plans']['keyset_limit_50']['execution_ms'] + offset = data['read_plans']['offset_50000_limit_50']['execution_ms'] + self.assertLess(keyset, offset, 'keyset should be faster than large OFFSET on hot chat') + self.assertIsNotNone(data['read_plans']['keyset_limit_50'].get('index_name')) + + def test_restore_artifact(self): + path = EVIDENCE / 'restore_report.json' + self.assertTrue(path.is_file(), 'missing restore_report.json') + data = json.loads(path.read_text(encoding='utf-8')) + self.assertTrue(data['counts_match']) + self.assertTrue(data['sample_match']) + self.assertGreaterEqual(data['source']['messages'], 100_000) + + +if __name__ == '__main__': + unittest.main()