18 changed files with 1332 additions and 32 deletions
+4 -9
View File
@@ -1,11 +1,6 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
POSTGRES_USER=messenger
POSTGRES_PASSWORD=change-me-before-running
POSTGRES_DB=messenger
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
DB_POOL_MAX=10
+24 -2
View File
@@ -12,5 +12,27 @@ jobs:
python-version: '3.12'
- run: python -m pip install -r requirements-dev.txt
- run: make check validate PYTHON=python
# This job validates the skeleton, not a student solution.
# Add build, readiness and make test in a separate application job.
application:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Prepare disposable local configuration
run: |
python - <<'PY'
from pathlib import Path
import secrets
config = Path('.env.example').read_text()
Path('.env').write_text(config.replace('change-me-before-running', secrets.token_urlsafe(24)))
Path('.env').chmod(0o600)
PY
- name: Build and wait for readiness
run: docker compose --env-file .env up -d --build --wait --wait-timeout 180
- name: Public API scenario
run: make test BASE_URL=http://127.0.0.1:8080 PYTHON=python
- name: Stop disposable runner stack
if: always()
run: docker compose --env-file .env down -v
+1
View File
@@ -30,3 +30,4 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
.pytest_cache/
+11
View File
@@ -0,0 +1,11 @@
FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_DEFAULT_TIMEOUT=60 PIP_RETRIES=3
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt && groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app
COPY --chown=app:app server.py ./server.py
COPY --chown=app:app migrations ./migrations
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=5 CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"
CMD ["python", "server.py"]
+13 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence
.PHONY: help check validate test load test-persistence test-ttl test-dependencies quality track2
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -24,3 +24,15 @@ load:
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
quality:
$(PYTHON) student-tests/quality.py --base-url "$(BASE_URL)"
test-ttl:
$(PYTHON) student-tests/session_ttl.py --base-url "$(BASE_URL)"
test-dependencies:
$(PYTHON) student-tests/dependencies.py --base-url "$(BASE_URL)"
track2:
$(PYTHON) scripts/track2_experiment.py
+56 -20
View File
@@ -1,37 +1,73 @@
# Отчёт — лабораторная 2
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Койчакаев Багавдин, БВТ2301.
- Целевая оценка: 5; трек: 2 — план запроса и восстановление.
- Ветка решения: `solution/bvt2301-koichakaev`, от `master` skeleton (`65003b61195361aac238f19c9bff0027ca58e09e`).
- Fork: https://gitea.privcloud.tech/koichakaev/mtusi2026lab2.
- PR: [ЛР 2 — БВТ2301 — Койчакаев — на 5, №139](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab2/pulls/139).
- Коммит реализации и результатов: [5b47c579cc](https://gitea.privcloud.tech/koichakaev/mtusi2026lab2/commit/5b47c579ccb4a730ac96a20e736f6d2cd8709cf7).
- Использованные источники, библиотеки и помощь ИИ: контракт и материалы курса; Python 3.12.8, psycopg 3.2.3, argon2-cffi 23.1.0, redis-py 5.2.1; при подготовке использовался Codex.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker Compose v2 и свободный локальный порт 8080. Скопировать `.env.example` в `.env`, заменить локальный `POSTGRES_PASSWORD`, затем:
```sh
cp .env.example .env
# Заменить POSTGRES_PASSWORD в .env на локальный пароль.
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait --wait-timeout 180
docker compose --env-file .env ps
make check
make test BASE_URL=http://127.0.0.1:8080
make track2
make quality BASE_URL=http://127.0.0.1:8080
make test-dependencies BASE_URL=http://127.0.0.1:8080
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080
```
Схема накатывается при старте API и отмечается в `schema_migrations`; повторный запуск не пересоздаёт таблицы. Стек использует `python:3.12.8-slim-bookworm` с digest, PostgreSQL `16.4-bookworm` и Valkey `8.0.1-alpine`; все Python-зависимости закреплены в `requirements.txt`. Остановка без удаления данных: `docker compose --env-file .env stop`. Для короткого TTL временно установить `SESSION_TTL_SECONDS=2`, пересоздать API (`docker compose --env-file .env up -d --force-recreate api`) и выполнить `make test-ttl`; после проверки вернуть TTL не менее 300 секунд. Удаление учебных данных выполняется только отдельной явной командой `docker compose --env-file .env down -v`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Из собственной lab1 сохранены HTTP API пользователей, чатов и сообщений, проверки входных данных, членства, пагинация и health endpoints. Хранилище заменено на PostgreSQL; вместо `X-User-Id` используется объявленный в lab2 переход на Basic/Bearer. Собственные дополнительные проверки находятся в `student-tests/`, исходные `tests/`, `contracts/`, `README.md` и `COURSE.md` сохранены без изменений. Файл `.github/workflows/contracts.yml` дополнен отдельным application job для сборки Compose, ожидания готовности и публичного smoke на одноразовом GitHub-hosted runner; локальные результаты приведены независимо от наличия CI на Gitea.
Схема: HTTP API → PostgreSQL; API также использует Valkey для TTL-кэша сессий. Источник истины для пользователей, чатов, membership, сообщений и сессий — PostgreSQL. Valkey хранит только короткоживущую связь хеша opaque-токена с user ID. PostgreSQL проверяет сессию при каждом запросе, поэтому logout отзывается и при недоступном Valkey; при отказе Valkey используется путь через PostgreSQL.
Пароли хешируются Argon2id через `argon2-cffi`; клиент получает случайный Bearer-токен с энтропией 256 бит, в PostgreSQL хранится только SHA-256 токена. Истёкшие токены отвергаются по `expires_at`. Бизнес-запросы принимают только Bearer; `X-User-Id` игнорируется. Пользователь, чат и его membership создаются транзакционно. История использует параметризованный keyset SQL по `(created_at,id)` и составной индекс `messages_history_keyset`; pool ограничен `DB_POOL_MAX`, ожидание соединения ограничено 3 секундами.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Критерий | Команда | Наблюдаемый результат |
| --- | --- | --- |
| Структура и контракт | `make check`; `docker compose --env-file .env config --quiet` | PASS |
| Публичный smoke | `make test BASE_URL=http://127.0.0.1:8080` | PASS: `lab2 public API scenario` |
| Сохранность | `make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080` | PASS: подтверждённые данные и сессия пережили рестарт API, PostgreSQL и Valkey с сохранёнными volumes |
| Гонки и транзакции | `make quality BASE_URL=http://127.0.0.1:8080` | PASS: 1×201 и 11×409 при 12 одинаковых регистрациях; невалидное создание чата не оставило строку |
| План истории | `make quality` после `make track2` | PASS: `Index Only Scan using messages_history_keyset`, 50 строк, 0,046 ms на таблице с 100k+ сообщений |
| Истечение и logout | `make test`; `make test-ttl` при TTL=2 | PASS: неверный пароль и logout проверены smoke; токен отвергнут после 2,2 s |
| Отказы зависимостей | `make test-dependencies BASE_URL=http://127.0.0.1:8080` | PASS: Valkey outage использовал PostgreSQL; PostgreSQL outage вернул 503 и восстановился с той же сессией |
| Трек 2 | `make track2` | PASS: 100k skewed сообщений, два индекса, три парных write trials и restore в отдельный PostgreSQL с отдельным volume; подробности в `evidence/track2-results.json` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Эксперимент: план запроса и восстановление
## Эксперименты и отказы
Гипотеза: индекс A `(chat_id,created_at,id)` будет компактнее; вариант B `(chat_id,created_at DESC,id DESC) INCLUDE(sender_id)` может снизить чтение страниц. B меняет направление сортировки двух ключей и добавляет sender_id: сравнение относится к двум вариантам в целом, а не к изолированному эффекту INCLUDE. Seed создаёт 100 000 синтетических сообщений: 80 000 в одном чате и 20 000 в 200 малых чатах, по 100 в каждом. Длина текста меняется от 40 до 199 символов. Детерминированные UUID и `ON CONFLICT` делают повторный запуск seed безопасным.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Для каждого индекса запускается одинаковый keyset-запрос на 50 строк с границей внутри большого чата. Затем проводятся три парных записи по 10 000 строк: каждый замер начинается с копии той же 100k таблицы, порядок A/B чередуется, построение индекса исключено из времени. PostgreSQL `EXPLAIN ANALYZE` измеряет исполнение вставки на стороне сервера. После этого `pg_dump -Fc` восстанавливается в отдельный PostgreSQL service с отдельным persistent volume `restore_data`; проверяются количества строк и первые десять выбранных benchmark-сообщений.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Среда: Docker Desktop 29.8.2, ARM64; Docker VM выделены 8 CPU и 4 106 604 544 байт RAM (около 3,82 GiB), отдельные container CPU/RAM limits не заданы. Runtime: Python 3.12.8, PostgreSQL 16.4, Valkey 8.0.1. План каждого индекса измерен один раз после seed и `ANALYZE`, без отдельного прогрева, поэтому различия в десятках микросекунд не считаются устойчивым выигрышем. Для write cost приведена медиана трёх парных повторов.
## Ограничения и следующие шаги
Результаты финального запуска, записанные в `evidence/track2-results.json` (для чтения указано Actual Total Time узла Limit; для записи — Execution Time):
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- A выбрал `Index Scan Forward`: 0,04 ms, shared hit=3, read=3; размер 5776 kB.
- B выбрал `Index Scan Backward`: 0,029 ms, shared hit=4, read=4; размер 7544 kB (+30,6%). Запрос выбирает `text`, поэтому `INCLUDE(sender_id)` не устраняет heap access и не даёт index-only scan. Единичные измерения не доказывают устойчивый выигрыш чтения.
- Медиана вставки 10k: A — 25,63 ms; B — 22,507 ms (-12.2%). На этих данных B увеличил размер индекса; трёх повторов недостаточно, чтобы отделить влияние индекса на время записи от шума среды. Для данного запроса выбран компактный индекс A.
- Backup занял 0,556 s и 6,882,247 bytes; restore — 0,608 s. В обеих БД совпали количества: 53 users, 214 chats, 227 chat_members, 100015 messages, 42 sessions. Первые десять выбранных synthetic messages совпали.
## Ограничения
Стенд использует HTTP и предназначен для loopback и синтетических данных. Репликация и failover не заявлены. Backup восстановлен в отдельный контейнер и volume того же Docker Desktop host; отказ всего host не моделировался, поэтому результаты не задают disaster recovery RPO/RTO. Плановые timings единичные, write trials — три; значения зависят от диска, CPU, RAM и прогрева PostgreSQL.
`pg_dump` фиксирует согласованный снимок БД: подтверждённые изменения после момента снимка в эту копию не попадут. Без архивирования WAL/PITR восстановление возможно только на момент последнего сохранённого backup; потенциальная потеря данных равна интервалу между его снимком и отказом. В эксперименте параллельная запись во время backup не выполнялась, поэтому сверка относится к неподвижному набору данных. Измеренные 0,608 s включают исполнение `pg_restore`, но не обнаружение аварии, передачу backup и подготовку нового сервера. Для отказа всего host копию нужно хранить отдельно от Docker Desktop.
Сводка полного прогона: [evidence/lab2-run.txt](evidence/lab2-run.txt). Исходные планы и измерения: [evidence/track2-results.json](evidence/track2-results.json). Вывод финальной проверки перед публикацией: [evidence/submission-checks.txt](evidence/submission-checks.txt).
+69
View File
@@ -0,0 +1,69 @@
services:
api:
build: .
restart: unless-stopped
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
CACHE_URL: redis://valkey:6379/0
DB_POOL_MAX: ${DB_POOL_MAX:-10}
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
depends_on:
postgres:
condition: service_healthy
valkey:
condition: service_healthy
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
interval: 10s
timeout: 3s
retries: 5
start_period: 20s
postgres:
image: postgres:16.4-bookworm
restart: unless-stopped
environment:
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
POSTGRES_DB: ${POSTGRES_DB:-messenger}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
postgres_restore:
image: postgres:16.4-bookworm
profiles: ["restore"]
restart: "no"
environment:
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
POSTGRES_DB: ${POSTGRES_DB:-messenger}
volumes:
- restore_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""]
interval: 5s
timeout: 3s
retries: 10
valkey:
image: valkey/valkey:8.0.1-alpine
restart: unless-stopped
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec"]
volumes:
- valkey_data:/data
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 5s
timeout: 3s
retries: 10
volumes:
postgres_data:
valkey_data:
restore_data:
+35
View File
@@ -0,0 +1,35 @@
Лабораторная 2 — фактический прогон
Дата: 2026-10-30
Среда: Docker Desktop 29.8.2; Docker VM ARM64, 8 CPU, 4 106 604 544 bytes RAM; лимиты отдельных сервисов не заданы.
Образы: python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0; postgres:16.4-bookworm; valkey/valkey:8.0.1-alpine.
Python packages в работающем API совпали с requirements.txt: argon2-cffi 23.1.0, bindings 26.1.0, cffi 2.1.1, pycparser 3.12, psycopg/binary 3.2.3, psycopg-pool 3.3.3, redis 5.2.1, typing_extensions 4.16.0.
make check
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
docker compose --env-file .env config --quiet
PASS (exit 0)
docker compose --env-file .env up -d --build
PASS; API, PostgreSQL and Valkey reported healthy. API published only on 127.0.0.1:8080.
make test BASE_URL=http://127.0.0.1:8080
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
make track2
PASS. Full plans, six write trials, row counts, backup and restore timings are in track2-results.json.
Seed: 100,000 messages; 80,000 in one large chat and 20,000 across 200 small chats.
Restore: separate PostgreSQL 16.4 process/service and persistent restore_data volume.
make quality BASE_URL=http://127.0.0.1:8080
PASS: registration race returned exactly 1 HTTP 201 and 11 HTTP 409; invalid chat creation left no chat row.
EXPLAIN (ANALYZE, BUFFERS): Index Only Scan using messages_history_keyset; 50 rows; Execution Time 0.046 ms; Heap Fetches 0.
make test-dependencies BASE_URL=http://127.0.0.1:8080
PASS: Valkey outage fell back to PostgreSQL; PostgreSQL outage returned bounded HTTP 503 and recovered with the same session.
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080
PASS: persistence preserved confirmed data and existing session after restarting API, PostgreSQL and Valkey; volumes retained.
make test-ttl BASE_URL=http://127.0.0.1:8080 (SESSION_TTL_SECONDS=2, temporary Compose env file)
PASS: session expired at the configured two-second TTL. The regular .env was restored to SESSION_TTL_SECONDS=3600 and the final smoke passed again.
+46
View File
@@ -0,0 +1,46 @@
2026-10-30 — финальная проверка оформления и переноса собственных тестов в student-tests
$ make check
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
Exit code: 0
$ docker compose --env-file .env config --quiet
Exit code: 0
$ make test BASE_URL=http://127.0.0.1:8080
python3 tests/smoke.py --base-url "http://127.0.0.1:8080"
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
Exit code: 0
$ make quality BASE_URL=http://127.0.0.1:8080
python3 student-tests/quality.py --base-url "http://127.0.0.1:8080"
{
"registration_race": {
"201": 1,
"409": 11
},
"chat_rollback": "PASS",
"history_index_plan": [
"Limit (cost=0.42..2.85 rows=50 width=24) (actual time=0.057..0.062 rows=50 loops=1)",
" Buffers: shared hit=2 read=2",
" -> Index Only Scan using messages_history_keyset on messages (cost=0.42..3894.60 rows=79896 width=24) (actual time=0.056..0.059 rows=50 loops=1)",
" Index Cond: (chat_id = 'f316dbbe-85f1-4587-454d-bfe607cf01de'::uuid)",
" Heap Fetches: 0",
" Buffers: shared hit=2 read=2",
"Planning:",
" Buffers: shared hit=110",
"Planning Time: 0.263 ms",
"Execution Time: 0.073 ms"
]
}
Exit code: 0
$ make test-dependencies BASE_URL=http://127.0.0.1:8080
python3 student-tests/dependencies.py --base-url "http://127.0.0.1:8080"
PASS: Valkey outage fell back to PostgreSQL; PostgreSQL outage returned bounded 503 and recovered with the same session
Exit code: 0
$ make track2
PASS: row counts and selected messages match; full plans and measured values saved in evidence/track2-results.json
Exit code: 0
+512
View File
@@ -0,0 +1,512 @@
{
"seed": "100000 messages, fixed seed/tag lab2_2026",
"restore_target": "separate PostgreSQL 16.4 service with its own restore_data volume",
"index_plans": [
{
"index": "A chat,created_at,id",
"total_runtime_ms": 0.04,
"shared_read_blocks": 3,
"shared_hit_blocks": 3,
"index_size": "5776 kB",
"plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 0.42,
"Total Cost": 13.59,
"Plan Rows": 50,
"Plan Width": 178,
"Actual Startup Time": 0.033,
"Actual Total Time": 0.04,
"Actual Rows": 50,
"Actual Loops": 1,
"Shared Hit Blocks": 3,
"Shared Read Blocks": 3,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "bench_messages_lab2_2026_a",
"Relation Name": "bench_messages_lab2_2026",
"Alias": "bench_messages_lab2_2026",
"Startup Cost": 0.42,
"Total Cost": 12636.91,
"Plan Rows": 47969,
"Plan Width": 178,
"Actual Startup Time": 0.032,
"Actual Total Time": 0.037,
"Actual Rows": 50,
"Actual Loops": 1,
"Index Cond": "((chat_id = 'f316dbbe-85f1-4587-454d-bfe607cf01de'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:40+00'::timestamp with time zone, '1791a97a-8403-730e-e076-0489a2aeb992'::uuid)))",
"Rows Removed by Index Recheck": 0,
"Shared Hit Blocks": 3,
"Shared Read Blocks": 3,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
},
{
"index": "B covering history index",
"total_runtime_ms": 0.029,
"shared_read_blocks": 4,
"shared_hit_blocks": 4,
"index_size": "7544 kB",
"plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 0.42,
"Total Cost": 14.03,
"Plan Rows": 50,
"Plan Width": 177,
"Actual Startup Time": 0.018,
"Actual Total Time": 0.029,
"Actual Rows": 50,
"Actual Loops": 1,
"Shared Hit Blocks": 4,
"Shared Read Blocks": 4,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Backward",
"Index Name": "bench_messages_lab2_2026_b",
"Relation Name": "bench_messages_lab2_2026",
"Alias": "bench_messages_lab2_2026",
"Startup Cost": 0.42,
"Total Cost": 13060.0,
"Plan Rows": 47981,
"Plan Width": 177,
"Actual Startup Time": 0.017,
"Actual Total Time": 0.027,
"Actual Rows": 50,
"Actual Loops": 1,
"Index Cond": "((chat_id = 'f316dbbe-85f1-4587-454d-bfe607cf01de'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:40+00'::timestamp with time zone, '1791a97a-8403-730e-e076-0489a2aeb992'::uuid)))",
"Rows Removed by Index Recheck": 0,
"Shared Hit Blocks": 4,
"Shared Read Blocks": 4,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
}
],
"write_cost": [
{
"trial": 1,
"index": "A",
"rows": 10000,
"execution_ms": 25.63,
"plan": {
"Node Type": "ModifyTable",
"Operation": "Insert",
"Parallel Aware": false,
"Async Capable": false,
"Relation Name": "bench_messages_lab2_2026_wa1",
"Alias": "bench_messages_lab2_2026_wa1",
"Startup Cost": 0.0,
"Total Cost": 350.0,
"Plan Rows": 0,
"Plan Width": 0,
"Actual Startup Time": 25.564,
"Actual Total Time": 25.565,
"Actual Rows": 0,
"Actual Loops": 1,
"Shared Hit Blocks": 41338,
"Shared Read Blocks": 5,
"Shared Dirtied Blocks": 249,
"Shared Written Blocks": 246,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Function Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Function Name": "generate_series",
"Alias": "g",
"Startup Cost": 0.0,
"Total Cost": 350.0,
"Plan Rows": 10000,
"Plan Width": 572,
"Actual Startup Time": 0.505,
"Actual Total Time": 10.841,
"Actual Rows": 10000,
"Actual Loops": 1,
"Shared Hit Blocks": 0,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
},
{
"trial": 1,
"index": "B",
"rows": 10000,
"execution_ms": 23.668,
"plan": {
"Node Type": "ModifyTable",
"Operation": "Insert",
"Parallel Aware": false,
"Async Capable": false,
"Relation Name": "bench_messages_lab2_2026_wb1",
"Alias": "bench_messages_lab2_2026_wb1",
"Startup Cost": 0.0,
"Total Cost": 350.0,
"Plan Rows": 0,
"Plan Width": 0,
"Actual Startup Time": 23.603,
"Actual Total Time": 23.603,
"Actual Rows": 0,
"Actual Loops": 1,
"Shared Hit Blocks": 41421,
"Shared Read Blocks": 6,
"Shared Dirtied Blocks": 291,
"Shared Written Blocks": 286,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Function Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Function Name": "generate_series",
"Alias": "g",
"Startup Cost": 0.0,
"Total Cost": 350.0,
"Plan Rows": 10000,
"Plan Width": 572,
"Actual Startup Time": 0.913,
"Actual Total Time": 10.025,
"Actual Rows": 10000,
"Actual Loops": 1,
"Shared Hit Blocks": 0,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
},
{
"trial": 2,
"index": "B",
"rows": 10000,
"execution_ms": 22.507,
"plan": {
"Node Type": "ModifyTable",
"Operation": "Insert",
"Parallel Aware": false,
"Async Capable": false,
"Relation Name": "bench_messages_lab2_2026_wb2",
"Alias": "bench_messages_lab2_2026_wb2",
"Startup Cost": 0.0,
"Total Cost": 350.0,
"Plan Rows": 0,
"Plan Width": 0,
"Actual Startup Time": 22.451,
"Actual Total Time": 22.451,
"Actual Rows": 0,
"Actual Loops": 1,
"Shared Hit Blocks": 41420,
"Shared Read Blocks": 6,
"Shared Dirtied Blocks": 291,
"Shared Written Blocks": 286,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Function Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Function Name": "generate_series",
"Alias": "g",
"Startup Cost": 0.0,
"Total Cost": 350.0,
"Plan Rows": 10000,
"Plan Width": 572,
"Actual Startup Time": 0.302,
"Actual Total Time": 9.352,
"Actual Rows": 10000,
"Actual Loops": 1,
"Shared Hit Blocks": 0,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
},
{
"trial": 2,
"index": "A",
"rows": 10000,
"execution_ms": 74.828,
"plan": {
"Node Type": "ModifyTable",
"Operation": "Insert",
"Parallel Aware": false,
"Async Capable": false,
"Relation Name": "bench_messages_lab2_2026_wa2",
"Alias": "bench_messages_lab2_2026_wa2",
"Startup Cost": 0.0,
"Total Cost": 350.0,
"Plan Rows": 0,
"Plan Width": 0,
"Actual Startup Time": 74.737,
"Actual Total Time": 74.738,
"Actual Rows": 0,
"Actual Loops": 1,
"Shared Hit Blocks": 41338,
"Shared Read Blocks": 5,
"Shared Dirtied Blocks": 249,
"Shared Written Blocks": 246,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Function Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Function Name": "generate_series",
"Alias": "g",
"Startup Cost": 0.0,
"Total Cost": 350.0,
"Plan Rows": 10000,
"Plan Width": 572,
"Actual Startup Time": 0.32,
"Actual Total Time": 16.837,
"Actual Rows": 10000,
"Actual Loops": 1,
"Shared Hit Blocks": 0,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
},
{
"trial": 3,
"index": "A",
"rows": 10000,
"execution_ms": 21.256,
"plan": {
"Node Type": "ModifyTable",
"Operation": "Insert",
"Parallel Aware": false,
"Async Capable": false,
"Relation Name": "bench_messages_lab2_2026_wa3",
"Alias": "bench_messages_lab2_2026_wa3",
"Startup Cost": 0.0,
"Total Cost": 350.0,
"Plan Rows": 0,
"Plan Width": 0,
"Actual Startup Time": 21.201,
"Actual Total Time": 21.202,
"Actual Rows": 0,
"Actual Loops": 1,
"Shared Hit Blocks": 41338,
"Shared Read Blocks": 5,
"Shared Dirtied Blocks": 249,
"Shared Written Blocks": 246,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Function Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Function Name": "generate_series",
"Alias": "g",
"Startup Cost": 0.0,
"Total Cost": 350.0,
"Plan Rows": 10000,
"Plan Width": 572,
"Actual Startup Time": 0.34,
"Actual Total Time": 9.433,
"Actual Rows": 10000,
"Actual Loops": 1,
"Shared Hit Blocks": 0,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
},
{
"trial": 3,
"index": "B",
"rows": 10000,
"execution_ms": 21.909,
"plan": {
"Node Type": "ModifyTable",
"Operation": "Insert",
"Parallel Aware": false,
"Async Capable": false,
"Relation Name": "bench_messages_lab2_2026_wb3",
"Alias": "bench_messages_lab2_2026_wb3",
"Startup Cost": 0.0,
"Total Cost": 350.0,
"Plan Rows": 0,
"Plan Width": 0,
"Actual Startup Time": 21.854,
"Actual Total Time": 21.855,
"Actual Rows": 0,
"Actual Loops": 1,
"Shared Hit Blocks": 41420,
"Shared Read Blocks": 6,
"Shared Dirtied Blocks": 291,
"Shared Written Blocks": 286,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Function Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Function Name": "generate_series",
"Alias": "g",
"Startup Cost": 0.0,
"Total Cost": 350.0,
"Plan Rows": 10000,
"Plan Width": 572,
"Actual Startup Time": 0.295,
"Actual Total Time": 9.481,
"Actual Rows": 10000,
"Actual Loops": 1,
"Shared Hit Blocks": 0,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
}
],
"backup_bytes": 6882247,
"backup_seconds": 0.556,
"restore_seconds": 0.608,
"row_counts": {
"messenger": {
"users": 53,
"chats": 214,
"chat_members": 227,
"messages": 100015,
"sessions": 42
},
"messenger_restore_check": {
"users": 53,
"chats": 214,
"chat_members": 227,
"messages": 100015,
"sessions": 42
}
},
"first_ten_rows_match": true
}
+39
View File
@@ -0,0 +1,39 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name varchar(100) NOT NULL CHECK (length(btrim(display_name)) > 0),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title varchar(100) NOT NULL CHECK (length(btrim(title)) > 0),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_by_user ON chat_members(user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
text varchar(4000) NOT NULL CHECK (length(btrim(text)) > 0),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX messages_history_keyset ON messages(chat_id, created_at, id);
-- PostgreSQL is authoritative for session durability and revocation. Valkey only
-- keeps a short-lived lookup hint; a cache hit is still checked against this row.
CREATE TABLE sessions (
token_hash char(64) PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX sessions_expiry ON sessions(expires_at);
+9
View File
@@ -0,0 +1,9 @@
argon2-cffi==23.1.0
argon2-cffi-bindings==26.1.0
cffi==2.1.1
pycparser==3.12
psycopg[binary,pool]==3.2.3
psycopg-binary==3.2.3
psycopg-pool==3.3.3
redis==5.2.1
typing_extensions==4.16.0
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
docker compose --env-file .env restart api postgres valkey
+94
View File
@@ -0,0 +1,94 @@
#!/usr/bin/env python3
"""Repeatable track 2 experiment: skewed 100k message history, two indexes, pg_dump/restore.
Run only on the local lab stack after `docker compose up -d`. It adds fixed
synthetic rows to the lab tables and uses disposable benchmark tables and a
separate restore database. Existing application rows and volumes are retained.
"""
import json
import os
import subprocess
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "evidence" / "track2-results.json"
DB = os.environ.get("POSTGRES_DB", "messenger")
USER = os.environ.get("POSTGRES_USER", "messenger")
RESTORE_DB = DB + "_restore_check"
def docker(*args, input=None):
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, input=input,
text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout
def sql(statement, database=DB, service="postgres"):
return docker("exec", "-T", service, "psql", "-X", "-v", "ON_ERROR_STOP=1", "-U", USER, "-d", database, "-At", "-c", statement)
def main():
tag = "lab2_2026"
table = "bench_messages_" + tag
# Seed one fixed synthetic user, one 80k-message room and 200 smaller rooms.
# Deterministic UUIDs and ON CONFLICT make reruns idempotent.
sql(f"""
INSERT INTO users(id,username,display_name,password_hash) VALUES (md5('user-{tag}')::uuid,'bench_{tag}','Benchmark user','not-a-login-hash') ON CONFLICT (username) DO NOTHING;
INSERT INTO chats(id,title) SELECT md5('chat-'||g)::uuid, 'Benchmark chat '||g FROM generate_series(0,200) g ON CONFLICT (id) DO NOTHING;
INSERT INTO chat_members(chat_id,user_id) SELECT md5('chat-'||g)::uuid,md5('user-{tag}')::uuid FROM generate_series(0,200) g ON CONFLICT DO NOTHING;
INSERT INTO messages(id,chat_id,sender_id,text,created_at)
SELECT md5('m'||g)::uuid,
CASE WHEN g<=80000 THEN md5('chat-0')::uuid ELSE md5('chat-'||((g-80001)%200+1))::uuid END,
md5('user-{tag}')::uuid,
repeat('x', 40 + (g%160)),
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 millisecond'
FROM generate_series(1,100000) AS g ON CONFLICT (id) DO NOTHING;
DROP TABLE IF EXISTS {table};
CREATE TABLE {table} (LIKE messages INCLUDING DEFAULTS);
INSERT INTO {table}(id,chat_id,sender_id,text,created_at)
SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE sender_id=md5('user-{tag}')::uuid;
ANALYZE {table};
CREATE INDEX {table}_a ON {table}(chat_id,created_at,id);
""")
samples = []
for label, index_sql in [
("A chat,created_at,id", f"CREATE INDEX {table}_a ON {table}(chat_id,created_at,id)"),
("B covering history index", f"CREATE INDEX {table}_b ON {table}(chat_id,created_at DESC,id DESC) INCLUDE(sender_id)")]:
if label.startswith("B"):
sql(f"DROP INDEX IF EXISTS {table}_a; {index_sql}; ANALYZE {table}")
plan = sql(f"EXPLAIN (ANALYZE,BUFFERS,FORMAT JSON) SELECT id,chat_id,sender_id,text,created_at FROM {table} WHERE chat_id=md5('chat-0')::uuid AND (created_at,id)>(timestamptz '2026-01-01 00:00:40+00',md5('cursor')::uuid) ORDER BY created_at,id LIMIT 50")
parsed = json.loads(plan)[0]["Plan"]
size = sql(f"SELECT pg_size_pretty(pg_relation_size('{table}_{'a' if label.startswith('A') else 'b'}'))")
samples.append({"index":label,"total_runtime_ms":parsed.get("Actual Total Time"),"shared_read_blocks":parsed.get("Shared Read Blocks"),"shared_hit_blocks":parsed.get("Shared Hit Blocks"),"index_size":size.strip(),"plan":parsed})
# Paired write-cost trials start with identical 100k-row tables. Index build
# and seed-copy time are excluded; order alternates to reduce warm-cache bias.
write = []
for trial in range(1, 4):
order = [("A", "a"), ("B", "b")] if trial % 2 else [("B", "b"), ("A", "a")]
for label, idx in order:
target = f"{table}_w{idx}{trial}"
index = (f"CREATE INDEX {target}_idx ON {target}(chat_id,created_at,id)" if idx == "a"
else f"CREATE INDEX {target}_idx ON {target}(chat_id,created_at DESC,id DESC) INCLUDE(sender_id)")
sql(f"CREATE TABLE {target} (LIKE {table} INCLUDING DEFAULTS); INSERT INTO {target} SELECT * FROM {table}; {index}")
write_plan=json.loads(sql(f"EXPLAIN (ANALYZE,BUFFERS,FORMAT JSON) INSERT INTO {target}(id,chat_id,sender_id,text,created_at) SELECT md5('write-{tag}-{label}-{trial}-'||g)::uuid, md5('chat-0')::uuid, md5('user-{tag}')::uuid, 'write sample', now()+g*interval '1 microsecond' FROM generate_series(1,10000) g"))[0]
write.append({"trial":trial,"index":label,"rows":10000,"execution_ms":write_plan["Execution Time"],"plan":write_plan["Plan"]})
sql(f"DROP TABLE {target}")
# Restore into a separate PostgreSQL process with its own persistent volume.
docker("--profile", "restore", "up", "-d", "--wait", "postgres_restore")
sql(f"DROP DATABASE IF EXISTS {RESTORE_DB}", database="postgres", service="postgres_restore")
sql(f"CREATE DATABASE {RESTORE_DB} OWNER {USER}", database="postgres", service="postgres_restore")
start=time.monotonic()
dump=subprocess.run(["docker","compose","--env-file",".env","exec","-T","postgres","pg_dump","-Fc","-U",USER,DB],cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,check=True).stdout
backup_seconds=time.monotonic()-start
start=time.monotonic()
subprocess.run(["docker","compose","--env-file",".env","exec","-T","postgres_restore","pg_restore","-U",USER,"-d",RESTORE_DB,"--no-owner","--no-privileges"],cwd=ROOT,input=dump,stdout=subprocess.PIPE,stderr=subprocess.PIPE,check=True)
restore_seconds=time.monotonic()-start
counts={DB:{name:int(sql(f"SELECT count(*) FROM {name}",database=DB).strip()) for name in ("users","chats","chat_members","messages","sessions")},RESTORE_DB:{name:int(sql(f"SELECT count(*) FROM {name}",database=RESTORE_DB,service="postgres_restore").strip()) for name in ("users","chats","chat_members","messages","sessions")}}
selected=sql(f"SELECT id::text||':'||text FROM messages WHERE sender_id=md5('user-{tag}')::uuid ORDER BY id LIMIT 10")
restored=sql(f"SELECT id::text||':'||text FROM messages WHERE sender_id=md5('user-{tag}')::uuid ORDER BY id LIMIT 10",database=RESTORE_DB,service="postgres_restore")
result={"seed":"100000 messages, fixed seed/tag "+tag,"restore_target":"separate PostgreSQL 16.4 service with its own restore_data volume","index_plans":samples,"write_cost":write,"backup_bytes":len(dump),"backup_seconds":round(backup_seconds,3),"restore_seconds":round(restore_seconds,3),"row_counts":counts,"first_ten_rows_match":selected==restored}
OUT.write_text(json.dumps(result,ensure_ascii=False,indent=2)+"\n",encoding="utf-8")
docker("stop", "postgres_restore")
if counts[DB] != counts[RESTORE_DB] or selected != restored:
raise RuntimeError("Restored row counts or selected messages differ from the source")
print(json.dumps(result,ensure_ascii=False,indent=2))
if __name__=="__main__": main()
+292
View File
@@ -0,0 +1,292 @@
#!/usr/bin/env python3
"""PostgreSQL-backed messenger API for MTUSI lab 2."""
from __future__ import annotations
import base64, binascii, hashlib, json, logging, os, re, secrets, signal, threading, time, uuid
from datetime import datetime, timedelta, timezone
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import parse_qs, urlsplit
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
import redis
from argon2 import PasswordHasher, exceptions as argon_exc
LOG = logging.getLogger("messenger")
USERNAME_RE = re.compile(r"\A[a-z0-9_]{3,32}\Z")
POOL: ConnectionPool | None = None
CACHE: redis.Redis | None = None
PASSWORDS = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
MAX_BODY_BYTES = 1024 * 1024
CURSOR_SECRET = secrets.token_bytes(32)
INSTANCE_ID = str(uuid.uuid4())
def utcnow() -> datetime: return datetime.now(timezone.utc)
def stamp(value: datetime) -> str: return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def new_id() -> str: return str(uuid.uuid4())
class ApiError(Exception):
def __init__(self, status, code, message): self.status, self.code, self.message = status, code, message
def invalid(message="Некорректный запрос"): raise ApiError(400, "invalid_request", message)
def unavailable(): raise ApiError(503, "unavailable", "Зависимость временно недоступна")
def db():
assert POOL is not None
return POOL.connection()
def migrate():
migration = Path(__file__).parent / "migrations" / "001_initial.sql"
with db() as conn:
conn.execute("SELECT pg_advisory_xact_lock(2026102802)")
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())")
if conn.execute("SELECT 1 FROM schema_migrations WHERE version='001_initial'").fetchone() is None:
conn.execute(migration.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations(version) VALUES ('001_initial')")
def as_user(row): return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": stamp(row["created_at"])}
def as_chat(row, members): return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in members], "created_at": stamp(row["created_at"])}
def as_message(row): return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": row["text"], "created_at": stamp(row["created_at"])}
def require_text(value, field, maximum):
if not isinstance(value, str) or not value.strip() or len(value) > maximum: invalid(f"Поле {field} должно содержать от 1 до {maximum} символов и не быть пустым")
try: value.encode("utf-8")
except UnicodeEncodeError: invalid(f"Поле {field} должно содержать корректный Unicode")
return value
def strict_object(value, allowed, required):
if not isinstance(value, dict) or set(value)-allowed or required-set(value): invalid("Тело запроса содержит неверные поля")
return value
def parse_uuid(value, label="UUID"):
try: return str(uuid.UUID(value))
except (ValueError, TypeError, AttributeError): invalid(f"Некорректный {label}")
def cursor_encode(scope, row):
obj = {"scope":scope,"created_at":stamp(row["created_at"]),"id":str(row["id"])}
raw = json.dumps(obj,separators=(",",":"),ensure_ascii=True).encode()
sig = hashlib.sha256(CURSOR_SECRET+raw).digest()[:12]
return base64.urlsafe_b64encode(raw+sig).decode().rstrip("=")
def cursor_decode(value, scope):
try:
raw = base64.urlsafe_b64decode(value+"="*(-len(value)%4)); payload,sig=raw[:-12],raw[-12:]
if not secrets.compare_digest(sig,hashlib.sha256(CURSOR_SECRET+payload).digest()[:12]): invalid("Некорректный cursor")
obj=json.loads(payload)
if set(obj)!={"scope","created_at","id"} or obj["scope"]!=scope: invalid("Cursor относится к другому списку")
dt=datetime.fromisoformat(obj["created_at"].replace("Z","+00:00")); return dt,parse_uuid(obj["id"])
except (ValueError,TypeError,KeyError,binascii.Error): invalid("Некорректный cursor")
class Handler(BaseHTTPRequestHandler):
protocol_version="HTTP/1.0"
def setup(self): super().setup(); self.connection.settimeout(3)
def log_message(self, fmt,*args): LOG.info("%s %s %s",self.command,urlsplit(self.path).path,args[1] if len(args)>1 else "")
def do_GET(self): self.dispatch()
def do_POST(self): self.dispatch()
def do_DELETE(self): self.dispatch()
def send_json(self,status,payload,headers=None):
body=json.dumps(payload,ensure_ascii=False,separators=(",",":")).encode()
self.send_response(status); self.send_header("Content-Type","application/json; charset=utf-8"); self.send_header("Content-Length",str(len(body)))
self.send_header("X-Request-Id",self.request_id); self.send_header("X-Instance-Id",INSTANCE_ID)
for k,v in (headers or {}).items(): self.send_header(k,v)
self.end_headers(); self.wfile.write(body)
def empty(self,status,headers=None):
self.send_response(status); self.send_header("Content-Length","0"); self.send_header("X-Request-Id",self.request_id); self.send_header("X-Instance-Id",INSTANCE_ID)
for k,v in (headers or {}).items(): self.send_header(k,v)
self.end_headers()
def error(self,e): self.send_json(e.status,{"error":{"code":e.code,"message":e.message,"request_id":self.request_id}},getattr(e,"headers",None))
def read_json(self):
if self.headers.get_content_type()!="application/json": raise ApiError(415,"unsupported_media_type","Ожидается Content-Type: application/json")
try: n=int(self.headers.get("Content-Length",""))
except ValueError: invalid("Некорректный Content-Length")
if n<0: invalid("Некорректный Content-Length")
if n>MAX_BODY_BYTES: raise ApiError(413,"payload_too_large","Тело запроса слишком велико")
raw=self.rfile.read(n)
if len(raw)!=n: invalid("Тело запроса передано не полностью")
try: val=json.loads(raw.decode("utf-8"))
except (UnicodeError,json.JSONDecodeError): invalid("Тело запроса должно быть корректным JSON UTF-8")
if not isinstance(val,dict): invalid("Тело запроса должно быть JSON-объектом")
return val
def challenge(self,scheme):
e=ApiError(401,"unauthorized","Нет действительной сессии"); e.headers={"WWW-Authenticate":scheme}; raise e
def authenticate(self):
header=self.headers.get("Authorization","")
if not header.startswith("Bearer "): self.challenge('Bearer realm="messenger"')
token=header[7:].strip()
if not token or len(token)>256: self.challenge('Bearer realm="messenger"')
digest=hashlib.sha256(token.encode()).hexdigest(); cache_key="sess:"+digest
uid=None
if CACHE:
try: uid=CACHE.get(cache_key)
except redis.RedisError: pass
with db() as conn:
# Valkey's expiring user ID is a lookup hint. PostgreSQL validates it
# against the live session on every request, so missed cache deletion
# during an outage can never keep a logged-out token valid.
row=None
if uid:
row=conn.execute("SELECT u.id,u.username,u.display_name,u.created_at,s.expires_at FROM sessions s JOIN users u ON u.id=s.user_id WHERE s.token_hash=%s AND s.user_id=%s AND s.expires_at>now()",(digest,uid)).fetchone()
if not row:
row=conn.execute("SELECT u.id,u.username,u.display_name,u.created_at,s.expires_at FROM sessions s JOIN users u ON u.id=s.user_id WHERE s.token_hash=%s AND s.expires_at>now()",(digest,)).fetchone()
if not row: self.challenge('Bearer realm="messenger"')
if CACHE and str(row["id"])!=uid:
try: CACHE.setex(cache_key,max(1,int((row["expires_at"]-utcnow()).total_seconds())),str(row["id"]))
except redis.RedisError: pass
return as_user(row),digest
def dispatch(self):
self.request_id=secrets.token_hex(16)
try:
split=urlsplit(self.path); path=split.path; q=parse_qs(split.query,keep_blank_values=True)
if path in ("/health/live","/health/ready"):
if self.command!="GET" or q: invalid("Health endpoint принимает только GET без параметров")
if path.endswith("ready"):
try:
with db() as c: c.execute("SELECT 1")
except Exception: unavailable()
self.send_json(200,{"status":"ok"}); return
if path=="/api/v1/users" and self.command=="POST": self.register(); return
if path=="/api/v1/auth/sessions" and self.command=="POST": self.login(); return
if path=="/api/v1/auth/sessions/current" and self.command=="DELETE":
actor,digest=self.authenticate()
with db() as c: c.execute("DELETE FROM sessions WHERE token_hash=%s",(digest,))
if CACHE:
try: CACHE.delete("sess:"+digest)
except redis.RedisError: pass
self.empty(204); return
actor,_=self.authenticate()
if path=="/api/v1/users/me" and self.command=="GET": self.no_query(q); self.send_json(200,actor); return
if path=="/api/v1/chats":
if self.command=="POST": self.no_query(q); self.create_chat(actor); return
if self.command=="GET": self.list_chats(actor,q); return
m=re.fullmatch(r"/api/v1/chats/([^/]+)(?:/(messages))?",path)
if m:
cid=parse_uuid(m.group(1),"chat_id")
if m.group(2)=="messages":
if self.command=="POST": self.create_message(actor,cid); return
if self.command=="GET": self.list_messages(actor,cid,q); return
if self.command=="GET":
self.no_query(q)
with db() as c:
row=c.execute("SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s",(cid,actor["id"])).fetchone()
if not row: raise ApiError(404,"not_found","Объект не найден")
ids=c.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id",(cid,)).fetchall()
self.send_json(200,as_chat(row,[x["user_id"] for x in ids])); return
raise ApiError(404,"not_found","Объект не найден")
except ApiError as e: self.error(e)
except (psycopg.Error,PoolTimeout,redis.RedisError) as e: LOG.warning("dependency error request_id=%s type=%s",self.request_id,type(e).__name__); self.error(ApiError(503,"unavailable","Зависимость временно недоступна"))
except (BrokenPipeError,ConnectionResetError): pass
except Exception: LOG.exception("request failed request_id=%s",self.request_id); self.error(ApiError(500,"internal_error","Внутренняя ошибка сервера"))
def no_query(self,q):
if q: invalid("Этот endpoint не принимает query-параметры")
def register(self):
b=strict_object(self.read_json(),{"username","display_name","password"},{"username","display_name","password"})
if not isinstance(b["username"],str) or not USERNAME_RE.fullmatch(b["username"]): invalid("username должен соответствовать [a-z0-9_]{3,32}")
name=require_text(b["display_name"],"display_name",100); password=b["password"]
if not isinstance(password,str) or not 12<=len(password)<=128: invalid("password должен содержать от 12 до 128 символов")
uid=new_id(); hashed=PASSWORDS.hash(password)
try:
with db() as c: row=c.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING id,username,display_name,created_at",(uid,b["username"],name,hashed)).fetchone()
except psycopg.errors.UniqueViolation: raise ApiError(409,"conflict","username уже занят")
self.send_json(201,as_user(row))
def login(self):
if self.headers.get("Content-Length") not in (None, "0", ""): invalid("POST /auth/sessions не принимает body")
h=self.headers.get("Authorization","")
try:
if not h.startswith("Basic "): raise ValueError()
userpass=base64.b64decode(h[6:],validate=True).decode("utf-8"); username,password=userpass.split(":",1)
except (ValueError,UnicodeError,binascii.Error): self.challenge('Basic realm="messenger", charset="UTF-8"')
with db() as c: row=c.execute("SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=%s",(username,)).fetchone()
# Verify a fixed dummy hash for unknown users to reduce timing-based enumeration.
try: ok=PASSWORDS.verify(row["password_hash"] if row else DUMMY_HASH,password)
except (argon_exc.VerificationError,ValueError): ok=False
if not row or not ok: self.challenge('Basic realm="messenger", charset="UTF-8"')
token=secrets.token_urlsafe(32); digest=hashlib.sha256(token.encode()).hexdigest(); ttl=int(os.getenv("SESSION_TTL_SECONDS","3600")); expires=utcnow()+timedelta(seconds=ttl)
with db() as c: c.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)",(digest,row["id"],expires))
if CACHE:
try: CACHE.setex("sess:"+digest,ttl,str(row["id"]))
except redis.RedisError: pass
self.send_json(201,{"token":token,"token_type":"Bearer","expires_at":stamp(expires),"user":as_user(row)})
def create_chat(self,actor):
b=strict_object(self.read_json(),{"title","member_ids"},{"title","member_ids"}); title=require_text(b["title"],"title",100); ids=b["member_ids"]
if not isinstance(ids,list) or not 1<=len(ids)<=100: invalid("member_ids должен содержать от 1 до 100 UUID")
ids=[parse_uuid(v,"member_id") for v in ids]
if len(set(ids))!=len(ids): invalid("member_ids не должен содержать повторов")
if actor["id"] not in ids: invalid("Создатель чата должен входить в member_ids")
cid=new_id()
try:
with db() as c:
known=c.execute("SELECT id FROM users WHERE id=ANY(%s)",(ids,)).fetchall()
if len(known)!=len(ids): invalid("Все участники чата должны быть зарегистрированы")
row=c.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at",(cid,title)).fetchone()
with c.cursor() as cursor:
cursor.executemany("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)",[(cid,u) for u in ids])
except psycopg.errors.ForeignKeyViolation: invalid("Все участники чата должны быть зарегистрированы")
self.send_json(201,as_chat(row,ids))
def parse_page(self,q,scope):
if set(q)-{"limit","cursor"} or any(len(v)!=1 for v in q.values()): invalid("Неверные параметры пагинации")
raw=q.get("limit",["50"])[0]
if not re.fullmatch(r"[0-9]{1,3}",raw): invalid("limit должен быть целым числом от 1 до 100")
limit=int(raw)
if not 1<=limit<=100: invalid("limit должен быть целым числом от 1 до 100")
cur=cursor_decode(q["cursor"][0],scope) if "cursor" in q and q["cursor"][0] else None
if "cursor" in q and not q["cursor"][0]: invalid("cursor не должен быть пустым")
return limit,cur
def list_chats(self,actor,q):
limit,cur=self.parse_page(q,"chats"); params=[actor["id"]]; clause=""
if cur: clause=" AND (c.created_at,c.id)>(%s,%s)"; params.extend(cur)
with db() as c:
rows=c.execute("SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s"+clause+" ORDER BY c.created_at,c.id LIMIT %s",(*params,limit+1)).fetchall()
items=[]
for row in rows[:limit]:
members=c.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id",(row["id"],)).fetchall(); items.append(as_chat(row,[m["user_id"] for m in members]))
self.send_json(200,{"items":items,"next_cursor":cursor_encode("chats",rows[limit-1]) if len(rows)>limit else None})
def create_message(self,actor,cid):
self.no_query(parse_qs(urlsplit(self.path).query,keep_blank_values=True))
with db() as c:
if not c.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",(cid,actor["id"])).fetchone(): raise ApiError(404,"not_found","Объект не найден")
b=strict_object(self.read_json(),{"text"},{"text"}); text=require_text(b["text"],"text",4000); mid=new_id()
try:
with db() as c: row=c.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING id,chat_id,sender_id,text,created_at",(mid,cid,actor["id"],text)).fetchone()
except psycopg.errors.ForeignKeyViolation: raise ApiError(404,"not_found","Объект не найден")
self.send_json(201,as_message(row))
def list_messages(self,actor,cid,q):
with db() as c:
if not c.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",(cid,actor["id"])).fetchone(): raise ApiError(404,"not_found","Объект не найден")
limit,cur=self.parse_page(q,"messages:"+cid); params=[cid]; clause=""
if cur: clause=" AND (created_at,id)>(%s,%s)"; params.extend(cur)
with db() as c: rows=c.execute("SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s"+clause+" ORDER BY created_at,id LIMIT %s",(*params,limit+1)).fetchall()
self.send_json(200,{"items":[as_message(r) for r in rows[:limit]],"next_cursor":cursor_encode("messages:"+cid,rows[limit-1]) if len(rows)>limit else None})
class Server(ThreadingHTTPServer):
daemon_threads=True; allow_reuse_address=True
def main():
global POOL,CACHE,DUMMY_HASH
logging.basicConfig(level=os.getenv("LOG_LEVEL","INFO").upper(),format="%(asctime)s %(levelname)s %(message)s")
pool_max=int(os.getenv("DB_POOL_MAX","10"))
if not 1<=pool_max<=50: raise SystemExit("DB_POOL_MAX must be between 1 and 50")
POOL=ConnectionPool(os.getenv("DATABASE_URL","postgresql://messenger:messenger@postgres:5432/messenger"),min_size=1,max_size=pool_max,timeout=3,kwargs={"row_factory":dict_row},open=True)
CACHE=redis.Redis.from_url(os.getenv("CACHE_URL","redis://valkey:6379/0"),socket_connect_timeout=0.5,socket_timeout=0.5,decode_responses=True)
DUMMY_HASH=PASSWORDS.hash("dummy-password-do-not-use-123")
deadline=time.monotonic()+60
while True:
try: migrate(); break
except (psycopg.Error, PoolTimeout):
if time.monotonic()>deadline: raise
LOG.info("waiting for PostgreSQL"); time.sleep(1)
try: CACHE.ping()
except redis.RedisError: LOG.warning("Valkey unavailable; session validation will use PostgreSQL")
host=os.getenv("APP_HOST","0.0.0.0"); port=int(os.getenv("APP_PORT","8080")); server=Server((host,port),Handler)
def stop(signum,_frame): LOG.info("received signal %s; stopping",signum); threading.Thread(target=server.shutdown,daemon=True).start()
signal.signal(signal.SIGTERM,stop); signal.signal(signal.SIGINT,stop)
try: LOG.info("server listening on %s:%s",host,port); server.serve_forever(poll_interval=.2)
finally: server.server_close(); POOL.close(); LOG.info("server stopped")
if __name__=="__main__": main()
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""Stop/start Valkey and PostgreSQL with volumes preserved; verify bounded failures and recovery."""
import subprocess
import time
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run
ROOT = Path(__file__).resolve().parents[1]
def compose(action, service):
subprocess.run(["docker", "compose", "--env-file", ".env", action, service], cwd=ROOT,
check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def main():
args=arguments("Check session fallback and database dependency failure")
c=Client(args.base_url,args.ca_file,timeout=8)
actor=c.user("dependency")
compose("stop","valkey")
try:
c.call("GET","/health/ready",200,schema="Health")
me=c.call("GET","/api/v1/users/me",200,actor=actor,schema="User")["json"]
check(me["id"]==actor["id"],"session failed while Valkey was down")
finally:
compose("start","valkey")
compose("stop","postgres")
try:
c.call("GET","/health/ready",503)
c.call("GET","/api/v1/users/me",503,actor=actor)
finally:
compose("start","postgres")
deadline=time.monotonic()+60
while True:
try:
c.call("GET","/health/ready",200,schema="Health")
break
except Failure:
check(time.monotonic()<deadline,"PostgreSQL did not recover within 60 seconds")
time.sleep(.5)
c.call("GET","/api/v1/users/me",200,actor=actor,schema="User")
print("PASS: Valkey outage fell back to PostgreSQL; PostgreSQL outage returned bounded 503 and recovered with the same session")
if __name__=="__main__": run(main)
+54
View File
@@ -0,0 +1,54 @@
#!/usr/bin/env python3
"""Level-4 checks: concurrent unique registration and atomic chat creation."""
import concurrent.futures
import json
import os
import subprocess
import uuid
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
def db_count(table):
# Fixed table allowlist: user input is never interpolated here.
assert table in {"users", "chats", "chat_members", "messages"}
output = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres",
"psql", "-X", "-U", "messenger", "-d", "messenger", "-At", "-c",
"SELECT count(*) FROM " + table], cwd=ROOT, check=True, text=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout
return int(output.strip())
def main():
args = arguments("Check registration race and transaction rollback")
c = Client(args.base_url, args.ca_file)
alice, bob, _, _ = c.bootstrap()
username = "race_" + uuid.uuid4().hex[:16]
body = {"username": username, "display_name": "Гонка регистрации", "password": "Test-password-" + uuid.uuid4().hex}
def register(_):
try:
return c.call("POST", "/api/v1/users", [201, 409], body=body)["status"]
except Failure:
raise
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
statuses = list(pool.map(register, range(12)))
check(statuses.count(201) == 1 and statuses.count(409) == 11,
"username race must create exactly one user and return 409 to all other requests")
before = db_count("chats")
after = c.call("POST", "/api/v1/chats", 400, actor=alice,
body={"title": "rollback probe", "member_ids": [alice["id"], str(uuid.uuid4())]})
check(after["status"] == 400 and db_count("chats") == before,
"invalid multi-step chat creation left a partial chat row")
plan = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres",
"psql", "-X", "-U", "messenger", "-d", "messenger", "-At", "-c",
"EXPLAIN (ANALYZE, BUFFERS) SELECT id FROM messages WHERE chat_id=md5('chat-0')::uuid ORDER BY created_at,id LIMIT 50"],
cwd=ROOT, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.splitlines()
print(json.dumps({"registration_race": {"201": statuses.count(201), "409": statuses.count(409)},
"chat_rollback": "PASS", "history_index_plan": plan}, ensure_ascii=False, indent=2))
if __name__ == "__main__": run(main)
+20
View File
@@ -0,0 +1,20 @@
#!/usr/bin/env python3
"""Run only with SESSION_TTL_SECONDS=2 in the Compose environment."""
import time
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run
def main():
args=arguments("Check persisted session expiry with a short TTL")
c=Client(args.base_url,args.ca_file)
actor=c.user("ttl")
c.call("GET","/api/v1/users/me",200,actor=actor,schema="User")
time.sleep(2.2)
c.call("GET","/api/v1/users/me",401,actor=actor)
print("PASS: session expired at the configured two-second TTL")
if __name__=="__main__": run(main)