From 7255878240566ae3eb636c43e5df28c85cbbda0d Mon Sep 17 00:00:00 2001 From: Yuliiiaaaa Date: Tue, 20 Oct 2026 00:04:12 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A02=20=E2=80=94=20=D0=91=D0=92=D0=A22?= =?UTF-8?q?302=20=E2=80=94=20=D0=9D=D0=B8=D0=BA=D0=B8=D1=82=D0=B8=D0=BD?= =?UTF-8?q?=D0=B0=20=E2=80=94=20=D0=BD=D0=B0=205=20(=D1=82=D1=80=D0=B5?= =?UTF-8?q?=D0=BA=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + 2026-01-01 | 0 Dockerfile | 10 + Makefile | 39 ++- REPORT.md | 51 +-- evidence/track2_explain_no_index.txt | 16 + evidence/track2_explain_with_index.txt | 16 + evidence/track2_restore.txt | 6 + evidence/track2_write_cost.txt | 22 ++ migrations/001_init.sql | 34 ++ requirements.txt | 7 + scripts/generate_data.sql | 41 +++ scripts/restart.bat | 4 + scripts/restart.example.sh | 10 - scripts/run_track1.ps1 | 79 +++++ scripts/run_track2.ps1 | 43 +++ src/app.py | 441 ++++++++++++++++++++++++ src/security.py | 16 + tests/student-tests/_bootstrap.py | 7 + tests/student-tests/load_experiment.py | 264 ++++++++++++++ tests/student-tests/run_all.sh | 8 + tests/student-tests/test_concurrency.py | 68 ++++ tests/student-tests/test_negative.py | 67 ++++ tests/student-tests/test_pagination.py | 69 ++++ tests/student-tests/test_sigterm.py | 58 ++++ tests/test_persistence.py | 95 +++++ 26 files changed, 1426 insertions(+), 47 deletions(-) create mode 100644 2026-01-01 create mode 100644 Dockerfile create mode 100644 evidence/track2_explain_no_index.txt create mode 100644 evidence/track2_explain_with_index.txt create mode 100644 evidence/track2_restore.txt create mode 100644 evidence/track2_write_cost.txt create mode 100644 migrations/001_init.sql create mode 100644 requirements.txt create mode 100644 scripts/generate_data.sql create mode 100644 scripts/restart.bat delete mode 100755 scripts/restart.example.sh create mode 100644 scripts/run_track1.ps1 create mode 100644 scripts/run_track2.ps1 create mode 100644 src/app.py create mode 100644 src/security.py create mode 100644 tests/student-tests/_bootstrap.py create mode 100644 tests/student-tests/load_experiment.py create mode 100644 tests/student-tests/run_all.sh create mode 100644 tests/student-tests/test_concurrency.py create mode 100644 tests/student-tests/test_negative.py create mode 100644 tests/student-tests/test_pagination.py create mode 100644 tests/student-tests/test_sigterm.py create mode 100644 tests/test_persistence.py diff --git a/.gitignore b/.gitignore index b5a71896..9f655299 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,7 @@ coverage/ .DS_Store .idea/ .vscode/ +backup.dump +*.dump # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. diff --git a/2026-01-01 b/2026-01-01 new file mode 100644 index 00000000..e69de29b diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..21b96262 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.7-slim-bookworm +RUN useradd -m -u 10001 appuser +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY src/ /app/src/ +COPY migrations/ /app/migrations/ +USER appuser +EXPOSE 8080 +CMD ["uvicorn", "src.app:app", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/Makefile b/Makefile index 9ecb223e..5a8451ff 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ -PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 -CA_FILE ?= -ACTION_SCRIPT ?= -TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) +PYTHON ?= python +BASE_URL ?= http://127.0.0.1:8080 +CA_FILE ?= +IMAGE ?= mtusi-messenger:lab2 +CONTAINER ?= mtusi-messenger-lab2 -.PHONY: help check validate test load test-persistence +.PHONY: help check validate test build run stop student-tests experiment test-persistence help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "targets: check validate test build run stop student-tests experiment test-persistence" check: $(PYTHON) scripts/check_contract.py @@ -16,11 +16,26 @@ validate: $(PYTHON) scripts/validate_openapi.py test: - $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PYTHON) tests/smoke.py --base-url $(BASE_URL) -load: - $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) +# Для Лабораторной 2 мы используем Docker Compose +build: + docker compose build + +run: + docker compose up -d + +stop: + docker compose down + +student-tests: + $(PYTHON) student-tests/test_negative.py --base-url $(BASE_URL) + $(PYTHON) student-tests/test_pagination.py --base-url $(BASE_URL) + $(PYTHON) student-tests/test_concurrency.py --base-url $(BASE_URL) + +experiment: + $(PYTHON) student-tests/load_experiment.py --base-url $(BASE_URL) \ + --history 1000 --concurrency 4 --duration 20 --output evidence/run.json test-persistence: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) - $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PYTHON) tests/test_persistence.py --base-url $(BASE_URL) --action-script $(ACTION_SCRIPT) \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index e42f2b05..fae2dac1 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,38 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Никитина Юлия Владимировна БВТ2302 +- Целевая оценка: 5 (трек 2: План запроса и восстановление) +- Commit/PR: https://gitflic.ru/project/vmd/mtusi2026lab1 (указать актуальную ссылку на MR) +- Помощь ИИ: генерация шаблонов текста Report, помощь в написании скриптов автоматизации для трека 2 +- Источники: FastAPI, psycopg3, redis-py, PostgreSQL 16, Redis 7. Сторонние библиотеки используются согласно `requirements.txt`. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования: Python 3.11+, Docker Desktop с Docker Compose. GNU make не обязателен (все команды можно выполнять напрямую). -## Архитектура и решения +```powershell +git clone && cd mtusi2026lab1-fork -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +# 1) Создание локального .env с паролями +@" +POSTGRES_PASSWORD=messenger_super_secret_password +SESSION_TTL_SECONDS=3600 +"@ | Out-File -FilePath .env -Encoding utf8 -## Доказательства критериев +# 2) Сборка и запуск всей инфраструктуры (API, PostgreSQL, Redis) +docker compose up -d --build -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +# 3) Ожидание готовности (БД должна применить миграцию) +Start-Sleep -Seconds 5 +curl.exe -s http://127.0.0.1:8080/health/ready +# Ожидается: {"status":"ok"} -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +# 4) Публичный smoke-тест +make test +# или python tests/smoke.py --base-url http://127.0.0.1:8080 -## Эксперименты и отказы +# 5) Проверка сохранности (Persistence) +make test-persistence ACTION_SCRIPT=".\scripts\restart.bat" -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | - -## Ограничения и следующие шаги - -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +# 6) Для оценки 5 (Трек 2): Сбор доказательств по плану запроса и восстановлению +powershell -ExecutionPolicy Bypass -File scripts/run_track2.ps1 \ No newline at end of file diff --git a/evidence/track2_explain_no_index.txt b/evidence/track2_explain_no_index.txt new file mode 100644 index 00000000..49b00588 --- /dev/null +++ b/evidence/track2_explain_no_index.txt @@ -0,0 +1,16 @@ + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=0.01..0.02 rows=1 width=88) (actual time=0.044..0.045 rows=0 loops=1) + Buffers: shared hit=3 + -> Sort (cost=0.01..0.02 rows=1 width=88) (actual time=0.042..0.043 rows=0 loops=1) + Sort Key: created_at + Sort Method: quicksort Memory: 25kB + Buffers: shared hit=3 + -> Seq Scan on messages (cost=0.00..0.00 rows=1 width=88) (actual time=0.010..0.011 rows=0 loops=1) + Filter: ((chat_id = '00000000-0000-0000-0000-000000000002'::uuid) AND (created_at > (now() - '50 days'::interval))) + Planning: + Buffers: shared hit=79 dirtied=1 + Planning Time: 0.771 ms + Execution Time: 0.086 ms +(12 rows) + diff --git a/evidence/track2_explain_with_index.txt b/evidence/track2_explain_with_index.txt new file mode 100644 index 00000000..5fae2007 --- /dev/null +++ b/evidence/track2_explain_with_index.txt @@ -0,0 +1,16 @@ + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=0.01..0.02 rows=1 width=88) (actual time=0.097..0.098 rows=0 loops=1) + Buffers: shared hit=3 + -> Sort (cost=0.01..0.02 rows=1 width=88) (actual time=0.095..0.096 rows=0 loops=1) + Sort Key: created_at + Sort Method: quicksort Memory: 25kB + Buffers: shared hit=3 + -> Seq Scan on messages (cost=0.00..0.00 rows=1 width=88) (actual time=0.019..0.019 rows=0 loops=1) + Filter: ((chat_id = '00000000-0000-0000-0000-000000000002'::uuid) AND (created_at > (now() - '50 days'::interval))) + Planning: + Buffers: shared hit=101 read=3 + Planning Time: 8.033 ms + Execution Time: 1.148 ms +(12 rows) + diff --git a/evidence/track2_restore.txt b/evidence/track2_restore.txt new file mode 100644 index 00000000..0d395450 --- /dev/null +++ b/evidence/track2_restore.txt @@ -0,0 +1,6 @@ +Restore time: 0.7619182 seconds + restored_messages +------------------- + 2000 +(1 row) + diff --git a/evidence/track2_write_cost.txt b/evidence/track2_write_cost.txt new file mode 100644 index 00000000..cbe452dd --- /dev/null +++ b/evidence/track2_write_cost.txt @@ -0,0 +1,22 @@ +DROP INDEX + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------ + Insert on messages (cost=0.00..22.50 rows=0 width=0) (actual time=12.555..12.557 rows=0 loops=1) + -> Function Scan on generate_series i (cost=0.00..22.50 rows=1000 width=88) (actual time=0.212..7.001 rows=1000 loops=1) + Planning Time: 2.225 ms + Trigger for constraint messages_chat_id_fkey: time=12.169 calls=1000 + Trigger for constraint messages_sender_id_fkey: time=11.114 calls=1000 + Execution Time: 39.576 ms +(6 rows) + +CREATE INDEX + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------ + Insert on messages (cost=0.00..22.50 rows=0 width=0) (actual time=17.904..17.906 rows=0 loops=1) + -> Function Scan on generate_series i (cost=0.00..22.50 rows=1000 width=88) (actual time=0.146..7.585 rows=1000 loops=1) + Planning Time: 0.126 ms + Trigger for constraint messages_chat_id_fkey: time=15.733 calls=1000 + Trigger for constraint messages_sender_id_fkey: time=15.707 calls=1000 + Execution Time: 49.733 ms +(6 rows) + diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..9c805d55 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,34 @@ +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + username VARCHAR(32) NOT NULL UNIQUE, + display_name VARCHAR(100) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT username_format CHECK (username ~ '^[a-z0-9_]{3,32}$') +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE messages ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +-- Критический индекс для keyset-пагинации и сортировки истории +CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id); \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..76f2efdb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib. +openapi-spec-validator==0.7.2 +fastapi==0.115.0 +uvicorn[standard]==0.30.6 +psycopg[binary,pool]==3.2.1 +redis==5.0.8 +pydantic==2.9.2 \ No newline at end of file diff --git a/scripts/generate_data.sql b/scripts/generate_data.sql new file mode 100644 index 00000000..d2c60ec5 --- /dev/null +++ b/scripts/generate_data.sql @@ -0,0 +1,41 @@ +-- Создаем тестового пользователя +INSERT INTO users (id, username, display_name, password_hash) +VALUES ('00000000-0000-0000-0000-000000000001', 'load_user', 'Load User', 'fake_hash') +ON CONFLICT DO NOTHING; + +-- Создаем большой чат (50 000 сообщений) +INSERT INTO chats (id, title) VALUES ('00000000-0000-0000-0000-000000000002', 'Big Chat'); +INSERT INTO chat_members (chat_id, user_id) VALUES ('00000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001'); + +-- Создаем средний чат (30 000 сообщений) +INSERT INTO chats (id, title) VALUES ('00000000-0000-0000-0000-000000000003', 'Medium Chat'); +INSERT INTO chat_members (chat_id, user_id) VALUES ('00000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000001'); + +-- Создаем маленький чат (20 000 сообщений) +INSERT INTO chats (id, title) VALUES ('00000000-0000-0000-0000-000000000004', 'Small Chat'); +INSERT INTO chat_members (chat_id, user_id) VALUES ('00000000-0000-0000-0000-000000000004', '00000000-0000-0000-0000-000000000001'); + +-- Генерируем сообщения +INSERT INTO messages (chat_id, sender_id, text, created_at) +SELECT + '00000000-0000-0000-0000-000000000002', + '00000000-0000-0000-0000-000000000001', + 'Big chat message ' || i, + now() - (100000 - i) * interval '1 second' +FROM generate_series(1, 50000) AS i; + +INSERT INTO messages (chat_id, sender_id, text, created_at) +SELECT + '00000000-0000-0000-0000-000000000003', + '00000000-0000-0000-0000-000000000001', + 'Medium chat message ' || i, + now() - (100000 - i) * interval '1 second' +FROM generate_series(1, 30000) AS i; + +INSERT INTO messages (chat_id, sender_id, text, created_at) +SELECT + '00000000-0000-0000-0000-000000000004', + '00000000-0000-0000-0000-000000000001', + 'Small chat message ' || i, + now() - (100000 - i) * interval '1 second' +FROM generate_series(1, 20000) AS i; \ No newline at end of file diff --git a/scripts/restart.bat b/scripts/restart.bat new file mode 100644 index 00000000..1e83dc1b --- /dev/null +++ b/scripts/restart.bat @@ -0,0 +1,4 @@ +@echo off +echo Restarting API, PostgreSQL, and Redis (volumes preserved)... +docker compose down +docker compose up -d \ No newline at end of file diff --git a/scripts/restart.example.sh b/scripts/restart.example.sh deleted file mode 100755 index 967e9512..00000000 --- a/scripts/restart.example.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -set -eu -# TODO: copy to restart.sh and implement using YOUR compose/stack services. -# Restart API and ALL used state stores (DB, sessions, S3, broker as applicable). -# Preserve persistent volumes and schema/data; never down -v / flush / re-seed. -# Allow operator unseal if required in lab7; do not print credentials. -# Exit 0 only after the intended action; test waits for API readiness separately. -printf '%s -' 'TODO: implement restart.sh for your infrastructure' >&2 -exit 2 diff --git a/scripts/run_track1.ps1 b/scripts/run_track1.ps1 new file mode 100644 index 00000000..30b7b2f3 --- /dev/null +++ b/scripts/run_track1.ps1 @@ -0,0 +1,79 @@ +param( + [string]$Image = "mtusi-messenger:lab1", + [string]$Container = "mtusi-messenger-lab1", + [string]$BaseUrl = "http://127.0.0.1:8080", + [string]$History = "1000,5000,20000", + [string]$Concurrency = "1,4,16", + [int]$Duration = 15, + [int]$Warmup = 2 +) + +$ErrorActionPreference = "Stop" + +function Stop-App { + $running = docker ps -q -f "name=$Container" + if ($running) { + Write-Host " stopping container..." + docker kill --signal SIGTERM $Container | Out-Null + for ($i = 0; $i -lt 60; $i++) { + $still = docker ps -q -f "name=$Container" + if (-not $still) { break } + Start-Sleep -Milliseconds 250 + } + } +} + +function Test-Ready { + $code = (& curl.exe -s -o NUL -m 2 -w "%{http_code}" "$BaseUrl/health/ready" 2>$null) + return ("$code".Trim() -eq "200") +} + +function Start-App([string]$Mode) { + docker run --rm -d --name $Container ` + -p 127.0.0.1:8080:8080 ` + -e HOST=0.0.0.0 -e PORT=8080 ` + -e PAGINATION_MODE=$Mode ` + -e MAX_HISTORY_PER_CHAT=100000 ` + -e MAX_INFLIGHT=256 ` + $Image | Out-Null + + for ($i = 0; $i -lt 80; $i++) { + if (Test-Ready) { + Write-Host " server ready (PAGINATION_MODE=$Mode)" + return + } + Start-Sleep -Milliseconds 300 + } + + Write-Host "---- diagnostics ----" + docker ps -a --filter "name=$Container" + Write-Host "---- last logs ----" + docker logs $Container --tail 30 + throw "Server did not become ready" +} + +# ------------- LINEAR ------------- +Write-Host "=== LINEAR ===" +Stop-App +Start-App "linear" +python student-tests\load_experiment.py ` + --base-url $BaseUrl ` + --mode linear --docker-container $Container ` + --history $History --concurrency $Concurrency ` + --duration $Duration --warmup $Warmup --read-ratio 0.8 ` + --output-dir evidence + +# ------------- INDEX ------------- +Write-Host "=== INDEX ===" +Stop-App +Start-App "index" +python student-tests\load_experiment.py ` + --base-url $BaseUrl ` + --mode index --docker-container $Container ` + --history $History --concurrency $Concurrency ` + --duration $Duration --warmup $Warmup --read-ratio 0.8 ` + --output-dir evidence + +Stop-App +Write-Host "" +Write-Host "DONE. See evidence\linear_summary.json and evidence\index_summary.json" \ No newline at end of file diff --git a/scripts/run_track2.ps1 b/scripts/run_track2.ps1 new file mode 100644 index 00000000..d07afbfe --- /dev/null +++ b/scripts/run_track2.ps1 @@ -0,0 +1,43 @@ +$ErrorActionPreference = "Stop" +$EvidDir = "evidence" +if (-not (Test-Path $EvidDir)) { New-Item -ItemType Directory -Path $EvidDir | Out-Null } + +Write-Host "=== TRACK 2: Gathering evidence ===" -ForegroundColor Cyan + +# 1. EXPLAIN WITH INDEX +Write-Host "1. Read query (WITH index)..." -ForegroundColor Yellow +$sql_with_idx = "EXPLAIN (ANALYZE, BUFFERS) SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '00000000-0000-0000-0000-000000000002' AND created_at > now() - interval '50 days' ORDER BY created_at ASC LIMIT 50;" +$sql_with_idx | docker compose exec -T db psql -U messenger -d messenger | Out-File -FilePath "$EvidDir\track2_explain_with_index.txt" -Encoding utf8 + +# 2. EXPLAIN WITHOUT INDEX +Write-Host "2. Read query (WITHOUT index)..." -ForegroundColor Yellow +"DROP INDEX idx_messages_chat_created;" | docker compose exec -T db psql -U messenger -d messenger | Out-Null +$sql_with_idx | docker compose exec -T db psql -U messenger -d messenger | Out-File -FilePath "$EvidDir\track2_explain_no_index.txt" -Encoding utf8 +"CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id);" | docker compose exec -T db psql -U messenger -d messenger | Out-Null + +# 3. WRITE COST +Write-Host "3. Write cost analysis..." -ForegroundColor Yellow +$insert_script = @" +DROP INDEX idx_messages_chat_created; +EXPLAIN ANALYZE INSERT INTO messages (chat_id, sender_id, text) SELECT '00000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001', 'test ' || i FROM generate_series(1, 1000) i; +CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id); +EXPLAIN ANALYZE INSERT INTO messages (chat_id, sender_id, text) SELECT '00000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001', 'test2 ' || i FROM generate_series(1, 1000) i; +"@ +$insert_script | docker compose exec -T db psql -U messenger -d messenger | Out-File -FilePath "$EvidDir\track2_write_cost.txt" -Encoding utf8 + +# 4. BACKUP +Write-Host "4. Creating backup..." -ForegroundColor Yellow +docker compose exec -T db pg_dump -U messenger -d messenger -Fc -f /tmp/backup.dump +docker cp mtusi2026lab1-fork-db-1:/tmp/backup.dump ./backup.dump | Out-Null + +# 5. RESTORE +Write-Host "5. Restoring from backup..." -ForegroundColor Yellow +docker compose exec -T db psql -U messenger -d postgres -c "DROP DATABASE IF EXISTS restored_db;" | Out-Null +docker compose exec -T db psql -U messenger -d postgres -c "CREATE DATABASE restored_db;" | Out-Null + +$time = Measure-Command { docker compose exec -T db pg_restore -U messenger -d restored_db -Fc /tmp/backup.dump } +"Restore time: $($time.TotalSeconds) seconds" | Out-File -FilePath "$EvidDir\track2_restore.txt" -Encoding utf8 +docker compose exec -T db psql -U messenger -d restored_db -c "SELECT COUNT(*) AS restored_messages FROM messages;" | Out-File -FilePath "$EvidDir\track2_restore.txt" -Encoding utf8 -Append + +Write-Host "=== DONE! All results saved to evidence/ ===" -ForegroundColor Green +Get-ChildItem $EvidDir \ No newline at end of file diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..a73ee7c1 --- /dev/null +++ b/src/app.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python3 +import base64 +import datetime as dt +import hashlib +import json +import os +import secrets +import uuid +from contextlib import asynccontextmanager +from typing import List, Optional + +import redis.asyncio as redis +from fastapi import FastAPI, Request, Response, HTTPException, Header, Depends +from fastapi.responses import JSONResponse +from psycopg_pool import AsyncConnectionPool +from pydantic import BaseModel, Field, ConfigDict + +# Configuration +DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@db:5432/messenger") +REDIS_URL = os.environ.get("REDIS_URL", "redis://redis:6379/0") +SESSION_TTL = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + +# Global state for pool and redis +pool: Optional[AsyncConnectionPool] = None +redis_client: Optional[redis.Redis] = None + + +# Pydantic Models (Contract Lab 2) +class UserCreate(BaseModel): + model_config = ConfigDict(extra='forbid') + username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(..., min_length=1, max_length=100) + password: str = Field(..., min_length=12, max_length=128) + +class UserOut(BaseModel): + id: str + username: str + display_name: str + created_at: str + +class ChatCreate(BaseModel): + model_config = ConfigDict(extra='forbid') + title: str = Field(..., min_length=1, max_length=100) + member_ids: List[str] = Field(..., min_length=1, max_length=100) + +class ChatOut(BaseModel): + id: str + title: str + member_ids: List[str] + created_at: str + +class MessageCreate(BaseModel): + model_config = ConfigDict(extra='forbid') + text: str = Field(..., min_length=1, max_length=4000) + +class MessageOut(BaseModel): + id: str + chat_id: str + sender_id: str + text: str + created_at: str + +class PageResponse(BaseModel): + items: List[dict] + next_cursor: Optional[str] + +class SessionResponse(BaseModel): + token: str + token_type: str = "Bearer" + expires_at: str + user: UserOut + + +# Security helpers +def hash_password(password: str) -> str: + """scrypt KDF with random salt.""" + salt = secrets.token_bytes(16) + # n=16384, r=8, p=1 is a good baseline for scrypt + key = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=64) + return f"{salt.hex()}${key.hex()}" + +def verify_password(password: str, stored: str) -> bool: + try: + salt_hex, key_hex = stored.split("$") + salt = bytes.fromhex(salt_hex) + expected_key = bytes.fromhex(key_hex) + actual_key = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=64) + return secrets.compare_digest(actual_key, expected_key) + except Exception: + return False + +def generate_token() -> str: + return secrets.token_urlsafe(48) + +def encode_cursor(scope: str, created_at: str) -> str: + payload = json.dumps({"s": scope, "t": created_at}, separators=(",", ":")) + return base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii").rstrip("=") + +def decode_cursor(cursor: str, expected_scope: str) -> Optional[str]: + try: + pad = "=" * (-len(cursor) % 4) + data = json.loads(base64.urlsafe_b64decode(cursor + pad)) + if data.get("s") == expected_scope and isinstance(data.get("t"), str): + return data["t"] + except Exception: + pass + return None + + +# FastAPI Lifespan (Startup / Shutdown) +@asynccontextmanager +async def lifespan(app: FastAPI): + global pool, redis_client + # Initialize DB pool + pool = AsyncConnectionPool(DATABASE_URL, min_size=2, max_size=10, open=False) + await pool.open() + # Initialize Redis + redis_client = redis.from_url(REDIS_URL, decode_responses=True) + + # Check connections + async with pool.connection() as conn: + await conn.execute("SELECT 1") + await redis_client.ping() + + yield + + # Cleanup + await pool.close() + await redis_client.aclose() + +app = FastAPI(lifespan=lifespan) + + +# Middleware & Error Handling +@app.middleware("http") +async def add_request_id(request: Request, call_next): + request.state.request_id = uuid.uuid4().hex + response = await call_next(request) + response.headers["X-Request-Id"] = request.state.request_id + return response + +@app.exception_handler(HTTPException) +async def custom_http_exception_handler(request: Request, exc: HTTPException): + # Map standard FastAPI detail to contract format + code_map = { + 400: "invalid_request", 401: "unauthorized", 404: "not_found", + 409: "conflict", 503: "unavailable", 500: "internal_error" + } + error_code = code_map.get(exc.status_code, "internal_error") + + payload = { + "error": { + "code": error_code, + "message": str(exc.detail), + "request_id": getattr(request.state, "request_id", "unknown") + } + } + return JSONResponse(status_code=exc.status_code, content=payload) + + +# Dependencies +async def get_current_user(request: Request, authorization: Optional[str] = Header(None)) -> dict: + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException(401, "Missing or invalid Authorization header") + + token = authorization[7:] + user_id = await redis_client.get(f"session:{token}") + if not user_id: + raise HTTPException(401, "Session expired or invalid") + + async with pool.connection() as conn: + async with conn.cursor() as cur: + await cur.execute( + "SELECT id::text, username, display_name, to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') FROM users WHERE id = %s", + (user_id,) + ) + row = await cur.fetchone() + if not row: + raise HTTPException(401, "User no longer exists") + return {"id": row[0], "username": row[1], "display_name": row[2], "created_at": row[3]} + + +# Health Endpoints +@app.get("/health/live") +async def health_live(): + return {"status": "ok"} + +@app.get("/health/ready") +async def health_ready(): + try: + async with pool.connection() as conn: + await conn.execute("SELECT 1") + await redis_client.ping() + return {"status": "ok"} + except Exception: + raise HTTPException(503, "Dependencies unavailable") + + +# Auth & Users +@app.post("/api/v1/users", status_code=201, response_model=UserOut) +async def create_user(body: UserCreate): + pwd_hash = hash_password(body.password) + async with pool.connection() as conn: + try: + async with conn.cursor() as cur: + await cur.execute( + "INSERT INTO users (username, display_name, password_hash) " + "VALUES (%s, %s, %s) " + "RETURNING id::text, username, display_name, to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')", + (body.username, body.display_name, pwd_hash) + ) + row = await cur.fetchone() + await conn.commit() + return {"id": row[0], "username": row[1], "display_name": row[2], "created_at": row[3]} + except Exception as e: + await conn.rollback() + if "unique" in str(e).lower(): + raise HTTPException(409, "Username already taken") + raise HTTPException(400, "Invalid user data") + +@app.post("/api/v1/auth/sessions", status_code=201, response_model=SessionResponse) +async def login(authorization: Optional[str] = Header(None)): + if not authorization or not authorization.startswith("Basic "): + raise HTTPException(401, "Basic authentication required") + + try: + creds = base64.b64decode(authorization[6:]).decode('utf-8').split(":", 1) + if len(creds) != 2: + raise ValueError() + username, password = creds + except Exception: + raise HTTPException(401, "Invalid credentials") + + async with pool.connection() as conn: + async with conn.cursor() as cur: + await cur.execute( + "SELECT id::text, password_hash, username, display_name, to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') FROM users WHERE username = %s", + (username,) + ) + row = await cur.fetchone() + + if not row or not verify_password(password, row[1]): + raise HTTPException(401, "Invalid credentials") + + token = generate_token() + await redis_client.setex(f"session:{token}", SESSION_TTL, row[0]) + + expires = (dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=SESSION_TTL)).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + return { + "token": token, + "token_type": "Bearer", + "expires_at": expires, + "user": {"id": row[0], "username": row[2], "display_name": row[3], "created_at": row[4]} + } + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def logout(authorization: Optional[str] = Header(None)): + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException(401, "Missing Authorization header") + token = authorization[7:] + + # Удаляем сессию из Redis + deleted = await redis_client.delete(f"session:{token}") + if not deleted: + raise HTTPException(401, "Session invalid or already logged out") + return Response(status_code=204) + +@app.get("/api/v1/users/me", response_model=UserOut) +async def get_me(user: dict = Depends(get_current_user)): + return user + + +# Chats & Messages +@app.post("/api/v1/chats", status_code=201, response_model=ChatOut) +async def create_chat(body: ChatCreate, user: dict = Depends(get_current_user)): + if user["id"] not in body.member_ids: + raise HTTPException(400, "Caller must be a member of the chat") + + unique_members = list(set(body.member_ids)) + if len(unique_members) != len(body.member_ids): + raise HTTPException(400, "Duplicate member IDs") + + async with pool.connection() as conn: + try: + async with conn.transaction(): + async with conn.cursor() as cur: + # Проверка, что все пользователи существуют + await cur.execute("SELECT id::text FROM users WHERE id = ANY(%s::uuid[])", (unique_members,)) + found_users = {r[0] for r in await cur.fetchall()} + if len(found_users) != len(unique_members): + raise HTTPException(400, "One or more members do not exist") + + # Создаём чат + await cur.execute( + "INSERT INTO chats (title) VALUES (%s) RETURNING id::text, title, to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')", + (body.title,) + ) + chat_row = await cur.fetchone() + chat_id = chat_row[0] + + # Добавляем участников + for uid in unique_members: + await cur.execute("INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", (chat_id, uid)) + + return {"id": chat_id, "title": chat_row[1], "member_ids": unique_members, "created_at": chat_row[2]} + except HTTPException: + raise + except Exception as e: + raise HTTPException(500, "Failed to create chat") + +@app.get("/api/v1/chats") +async def list_chats(limit: int = 50, cursor: Optional[str] = None, user: dict = Depends(get_current_user)): + if not (1 <= limit <= 100): + raise HTTPException(400, "Invalid limit") + + cursor_ts = None + if cursor: + cursor_ts = decode_cursor(cursor, "chats") + if not cursor_ts: + raise HTTPException(400, "Invalid cursor") + + async with pool.connection() as conn: + async with conn.cursor() as cur: + if cursor_ts: + await cur.execute( + "SELECT c.id::text, c.title, to_char(c.created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') " + "FROM chats c JOIN chat_members cm ON c.id = cm.chat_id " + "WHERE cm.user_id = %s AND c.created_at > %s " + "ORDER BY c.created_at ASC LIMIT %s", + (user["id"], cursor_ts, limit + 1) + ) + else: + await cur.execute( + "SELECT c.id::text, c.title, to_char(c.created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') " + "FROM chats c JOIN chat_members cm ON c.id = cm.chat_id " + "WHERE cm.user_id = %s " + "ORDER BY c.created_at ASC LIMIT %s", + (user["id"], limit + 1) + ) + rows = await cur.fetchall() + + # Получаем member_ids для каждого чата + chat_ids = [r[0] for r in rows[:limit]] + members_map = {} + if chat_ids: + await cur.execute("SELECT chat_id::text, user_id::text FROM chat_members WHERE chat_id = ANY(%s::uuid[])", (chat_ids,)) + for cid, uid in await cur.fetchall(): + members_map.setdefault(cid, []).append(uid) + + items = [] + next_cursor = None + for r in rows[:limit]: + items.append({"id": r[0], "title": r[1], "created_at": r[2], "member_ids": members_map.get(r[0], [])}) + + if len(rows) > limit: + next_cursor = encode_cursor("chats", items[-1]["created_at"]) + + return {"items": items, "next_cursor": next_cursor} + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, user: dict = Depends(get_current_user)): + async with pool.connection() as conn: + async with conn.cursor() as cur: + # Проверка членства + await cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user["id"])) + if not await cur.fetchone(): + raise HTTPException(404, "Chat not found") + + await cur.execute( + "SELECT id::text, title, to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') FROM chats WHERE id = %s", + (chat_id,) + ) + chat = await cur.fetchone() + + await cur.execute("SELECT user_id::text FROM chat_members WHERE chat_id = %s", (chat_id,)) + members = [r[0] for r in await cur.fetchall()] + + return {"id": chat[0], "title": chat[1], "created_at": chat[2], "member_ids": members} + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201, response_model=MessageOut) +async def create_message(chat_id: str, body: MessageCreate, user: dict = Depends(get_current_user)): + async with pool.connection() as conn: + async with conn.cursor() as cur: + await cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user["id"])) + if not await cur.fetchone(): + raise HTTPException(404, "Chat not found") + + await cur.execute( + "INSERT INTO messages (chat_id, sender_id, text) " + "VALUES (%s, %s, %s) " + "RETURNING id::text, chat_id::text, sender_id::text, text, to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')", + (chat_id, user["id"], body.text) + ) + row = await cur.fetchone() + await conn.commit() + return {"id": row[0], "chat_id": row[1], "sender_id": row[2], "text": row[3], "created_at": row[4]} + +@app.get("/api/v1/chats/{chat_id}/messages", response_model=PageResponse) +async def list_messages(chat_id: str, limit: int = 50, cursor: Optional[str] = None, user: dict = Depends(get_current_user)): + if not (1 <= limit <= 100): + raise HTTPException(400, "Invalid limit") + + scope = f"chat:{chat_id}" + cursor_ts = None + if cursor: + cursor_ts = decode_cursor(cursor, scope) + if not cursor_ts: + raise HTTPException(400, "Invalid cursor") + + async with pool.connection() as conn: + async with conn.cursor() as cur: + await cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user["id"])) + if not await cur.fetchone(): + raise HTTPException(404, "Chat not found") + + query = ( + "SELECT id::text, chat_id::text, sender_id::text, text, to_char(created_at, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') " + "FROM messages WHERE chat_id = %s " + ) + params = [chat_id] + + if cursor_ts: + query += "AND created_at > %s " + params.append(cursor_ts) + + query += "ORDER BY created_at ASC LIMIT %s" + params.append(limit + 1) + + await cur.execute(query, tuple(params)) + rows = await cur.fetchall() + + items = [] + next_cursor = None + for r in rows[:limit]: + items.append({"id": r[0], "chat_id": r[1], "sender_id": r[2], "text": r[3], "created_at": r[4]}) + + if len(rows) > limit: + next_cursor = encode_cursor(scope, items[-1]["created_at"]) + + return {"items": items, "next_cursor": next_cursor} \ No newline at end of file diff --git a/src/security.py b/src/security.py new file mode 100644 index 00000000..6bbdd78b --- /dev/null +++ b/src/security.py @@ -0,0 +1,16 @@ +import hashlib, os, secrets + +def hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + key = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=64) + return salt.hex() + "$" + key.hex() + +def verify_password(password: str, stored: str) -> bool: + salt_hex, key_hex = stored.split("$") + salt = bytes.fromhex(salt_hex) + expected = bytes.fromhex(key_hex) + actual = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=64) + return secrets.compare_digest(actual, expected) + +def generate_token() -> str: + return secrets.token_urlsafe(48) # ~384 бита энтропии \ No newline at end of file diff --git a/tests/student-tests/_bootstrap.py b/tests/student-tests/_bootstrap.py new file mode 100644 index 00000000..7850e011 --- /dev/null +++ b/tests/student-tests/_bootstrap.py @@ -0,0 +1,7 @@ +"""Shared path bootstrap so student tests can import tests/client.py.""" +import sys +from pathlib import Path + +TESTS_DIR = Path(__file__).resolve().parents[1] / "tests" +if str(TESTS_DIR) not in sys.path: + sys.path.insert(0, str(TESTS_DIR)) \ No newline at end of file diff --git a/tests/student-tests/load_experiment.py b/tests/student-tests/load_experiment.py new file mode 100644 index 00000000..050bb06b --- /dev/null +++ b/tests/student-tests/load_experiment.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +"""Track 1: measure in-memory architecture limits. + +Closed-loop workload with a mix of paginated reads (random cursor depth) +and writes. Iterates over history sizes and concurrency levels in one +process, so the server is seeded once per history size. + +Requires the server to already run with the desired PAGINATION_MODE. +Use --docker-container to sample RSS via ``docker stats`` (Windows friendly). +""" +import argparse +import json +import math +import random +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, Failure # noqa: E402 + + +def pct(sorted_values, p): + if not sorted_values: + return None + idx = max(0, math.ceil(p * len(sorted_values)) - 1) + return round(1000 * sorted_values[idx], 3) + + +def read_rss_kb(pid): + try: + text = Path("/proc/%d/status" % pid).read_text() + except OSError: + return None + for line in text.splitlines(): + if line.startswith("VmRSS:"): + return int(line.split()[1]) + return None + + +def docker_rss_kb(container): + """Return MiB/KiB value from `docker stats --no-stream` as KB.""" + try: + out = subprocess.run( + ["docker", "stats", "--no-stream", + "--format", "{{.MemUsage}}", container], + capture_output=True, text=True, timeout=15).stdout.strip() + except Exception: + return None + if not out or "/" not in out: + return None + used = out.split("/")[0].strip() + for suffix, factor in (("GiB", 1024 * 1024), ("MiB", 1024), + ("KiB", 1), ("B", 1 / 1024)): + if used.endswith(suffix): + try: + return int(float(used[:-len(suffix)]) * factor) + except ValueError: + return None + return None + + +def sample_rss(args): + if args.server_pid: + return read_rss_kb(args.server_pid) + if args.docker_container: + return docker_rss_kb(args.docker_container) + return None + + +def prepopulate(client, actor, chat, count, existing, progress_every=2000): + if existing >= count: + return existing + t0 = time.monotonic() + for i in range(existing, count): + client.message(actor, chat, "seed-%08d" % i) + if progress_every and (i + 1) % progress_every == 0: + print(" seeded %d/%d (%.1fs)" + % (i + 1, count, time.monotonic() - t0), + file=sys.stderr) + return count + + +def collect_cursors(client, actor, chat, page_limit=50): + cursors = [] + path = "/api/v1/chats/%s/messages?limit=%d" % (chat["id"], page_limit) + while True: + page = client.call("GET", path, actor=actor, + schema="MessagePage")["json"] + nxt = page["next_cursor"] + if nxt is None: + break + cursors.append(nxt) + path = "/api/v1/chats/%s/messages?limit=%d&cursor=%s" % ( + chat["id"], page_limit, nxt) + return cursors + + +def op_once(client, actor, chat, cursors, rng, read_ratio): + if cursors and rng.random() < read_ratio: + cur = rng.choice(cursors) + start = time.monotonic() + client.call("GET", + "/api/v1/chats/%s/messages?limit=50&cursor=%s" + % (chat["id"], cur), + actor=actor, schema="MessagePage") + return "read", time.monotonic() - start + start = time.monotonic() + client.message(actor, chat, "load-" + uuid.uuid4().hex[:8]) + return "write", time.monotonic() - start + + +def run_one(args, alice, chat, history, cursors, concurrency): + barrier = threading.Barrier(concurrency) + results = [None] * concurrency + + def worker(wid): + local = Client(args.base_url, timeout=15) + rng = random.Random(0xC0FFEE + wid) + read_lat, write_lat = [], [] + read_err = write_err = 0 + w_deadline = time.monotonic() + args.warmup + while time.monotonic() < w_deadline: + try: + op_once(local, alice, chat, cursors, rng, args.read_ratio) + except Failure: + pass + barrier.wait() + deadline = time.monotonic() + args.duration + while time.monotonic() < deadline: + try: + kind, dt_s = op_once(local, alice, chat, cursors, + rng, args.read_ratio) + except Failure: + if rng.random() < args.read_ratio: + read_err += 1 + else: + write_err += 1 + continue + (read_lat if kind == "read" else write_lat).append(dt_s) + results[wid] = (read_lat, write_lat, read_err, write_err) + + started = time.monotonic() + threads = [threading.Thread(target=worker, args=(i,)) + for i in range(concurrency)] + for t in threads: + t.start() + for t in threads: + t.join() + elapsed = time.monotonic() - started + + read_all, write_all = [], [] + read_err_total = write_err_total = 0 + for chunk in results: + r_lat, w_lat, r_err, w_err = chunk + read_all.extend(r_lat) + write_all.extend(w_lat) + read_err_total += r_err + write_err_total += w_err + read_all.sort() + write_all.sort() + + return { + "history": history, + "concurrency": concurrency, + "duration_s": args.duration, + "elapsed_s": round(elapsed, 3), + "read_ratio": args.read_ratio, + "cursors": len(cursors), + "reads": { + "success": len(read_all), + "errors": read_err_total, + "rps": round(len(read_all) / elapsed, 2), + "p50_ms": pct(read_all, 0.50), + "p95_ms": pct(read_all, 0.95), + "p99_ms": pct(read_all, 0.99), + }, + "writes": { + "success": len(write_all), + "errors": write_err_total, + "rps": round(len(write_all) / elapsed, 2), + "p50_ms": pct(write_all, 0.50), + "p95_ms": pct(write_all, 0.95), + "p99_ms": pct(write_all, 0.99), + }, + "total_ops": len(read_all) + len(write_all) + read_err_total + write_err_total, + "error_ratio": round( + (read_err_total + write_err_total) / + max(1, len(read_all) + len(write_all) + read_err_total + write_err_total), + 6), + } + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--history", default="1000,5000,20000", + help="comma-separated history sizes") + ap.add_argument("--concurrency", default="1,4,16", + help="comma-separated concurrency levels") + ap.add_argument("--duration", type=float, default=15.0) + ap.add_argument("--warmup", type=float, default=2.0) + ap.add_argument("--read-ratio", type=float, default=0.8) + ap.add_argument("--server-pid", type=int, default=None, + help="server PID for /proc RSS sampling (Linux)") + ap.add_argument("--docker-container", default=None, + help="container name for `docker stats` RSS sampling") + ap.add_argument("--mode", default=None, help="label: linear | index") + ap.add_argument("--output-dir", default="evidence") + args = ap.parse_args() + + histories = [int(x) for x in args.history.split(",") if x.strip()] + concurrencies = [int(x) for x in args.concurrency.split(",") if x.strip()] + label = args.mode or "run" + + c = Client(args.base_url) + alice, bob, eve, chat = c.bootstrap() + print("chat=%s" % chat["id"], file=sys.stderr) + + out_dir = Path(args.output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + seeded = 0 + summaries = [] + for history in histories: + print("== history=%d ==" % history, file=sys.stderr) + seeded = prepopulate(c, alice, chat, history, seeded) + cursors = collect_cursors(c, bob, chat) + rss_after_seed = sample_rss(args) + print(" cursors=%d rss_kb=%s" + % (len(cursors), rss_after_seed), file=sys.stderr) + for concurrency in concurrencies: + print(" -- concurrency=%d" % concurrency, file=sys.stderr) + r = run_one(args, alice, chat, history, cursors, concurrency) + r["rss_kb_after_seed"] = rss_after_seed + r["rss_kb_end"] = sample_rss(args) + r["mode"] = label + summaries.append(r) + out = out_dir / ("%s_h%d_c%d.json" + % (label, history, concurrency)) + out.write_text(json.dumps(r, indent=2) + "\n") + print(json.dumps({ + "mode": r["mode"], "history": r["history"], + "concurrency": r["concurrency"], + "read_rps": r["reads"]["rps"], + "read_p95_ms": r["reads"]["p95_ms"], + "read_p99_ms": r["reads"]["p99_ms"], + "write_rps": r["writes"]["rps"], + "write_p95_ms": r["writes"]["p95_ms"], + "write_p99_ms": r["writes"]["p99_ms"], + "err_ratio": r["error_ratio"], + "rss_kb_end": r["rss_kb_end"], + })) + + combined = out_dir / ("%s_summary.json" % label) + combined.write_text(json.dumps(summaries, indent=2) + "\n") + print("combined summary: %s" % combined, file=sys.stderr) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/student-tests/run_all.sh b/tests/student-tests/run_all.sh new file mode 100644 index 00000000..d72830d0 --- /dev/null +++ b/tests/student-tests/run_all.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +BASE_URL="${BASE_URL:-http://localhost:8080}" +PY="${PY:-python3}" + +"$PY" student-tests/test_negative.py --base-url "$BASE_URL" +"$PY" student-tests/test_pagination.py --base-url "$BASE_URL" +"$PY" student-tests/test_concurrency.py --base-url "$BASE_URL" \ No newline at end of file diff --git a/tests/student-tests/test_concurrency.py b/tests/student-tests/test_concurrency.py new file mode 100644 index 00000000..0aa9bda0 --- /dev/null +++ b/tests/student-tests/test_concurrency.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""100 parallel POSTs from 10 clients -> exactly 100 unique messages.""" +import threading +from _bootstrap import TESTS_DIR # noqa: F401 + +from client import Client, arguments, check, run + + +def main(): + args = arguments('Student concurrency test') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + + workers, per_worker = 10, 10 + barrier = threading.Barrier(workers) + ids_per_worker = [None] * workers + errors = [None] * workers + + def worker(wid): + local = Client(args.base_url, args.ca_file, timeout=20) + barrier.wait() + collected = [] + try: + for i in range(per_worker): + m = local.message(alice, chat, 'w%02d-%02d' % (wid, i))['json'] + collected.append(m['id']) + except Exception as exc: # noqa: BLE001 + errors[wid] = repr(exc) + ids_per_worker[wid] = collected + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(workers)] + for t in threads: + t.start() + for t in threads: + t.join() + + failures = [e for e in errors if e] + check(not failures, 'worker errors: ' + '; '.join(failures)) + + all_ids = [mid for chunk in ids_per_worker for mid in chunk] + check(len(all_ids) == workers * per_worker, + 'expected %d writes, got %d' % (workers * per_worker, len(all_ids))) + check(len(set(all_ids)) == len(all_ids), 'duplicate message ids returned') + + # Read back the entire history + seen = set() + cursor = None + pages = 0 + while True: + q = path + '/messages?limit=100' + if cursor: + q += '&cursor=' + cursor + page = c.call('GET', q, actor=bob, schema='MessagePage')['json'] + for item in page['items']: + seen.add(item['id']) + pages += 1 + if page['next_cursor'] is None or pages > 50: + break + cursor = page['next_cursor'] + + missing = [mid for mid in all_ids if mid not in seen] + check(not missing, 'history lost %d messages' % len(missing)) + print('PASS: 100 messages from 10 clients, all unique and readable') + + +if __name__ == '__main__': + run(main) \ No newline at end of file diff --git a/tests/student-tests/test_negative.py b/tests/student-tests/test_negative.py new file mode 100644 index 00000000..1b800172 --- /dev/null +++ b/tests/student-tests/test_negative.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Negative-path checks: bad JSON, empty/oversized text, foreign chat, +duplicate username, bad auth, invalid limit, extra fields.""" +from _bootstrap import TESTS_DIR # noqa: F401 (adds tests/ to sys.path) +import uuid + +from client import Client, arguments, run + + +def main(): + args = arguments('Student negative tests') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + + # 1. Malformed JSON: the client asserts the server must answer 400. + c.call('POST', '/api/v1/users', 400, + raw=b'{not json', + headers={'Content-Type': 'application/json'}) + c.call('POST', path + '/messages', 400, actor=alice, + raw=b'{"text":', + headers={'Content-Type': 'application/json'}) + + # 2. Empty / whitespace / oversized text. + c.call('POST', path + '/messages', 400, actor=alice, body={'text': ''}) + c.call('POST', path + '/messages', 400, actor=alice, body={'text': ' '}) + c.call('POST', path + '/messages', 400, actor=alice, + body={'text': 'x' * 4001}) + + # 3. Foreign chat: eve is not a member. + c.call('GET', path, 404, actor=eve) + c.call('POST', path + '/messages', 404, actor=eve, body={'text': 'hack'}) + + # 4. Duplicate username. + dup = {'username': 'uniq_' + uuid.uuid4().hex[:8], 'display_name': 'Dup'} + c.call('POST', '/api/v1/users', 201, body=dup) + c.call('POST', '/api/v1/users', 409, body=dup) + + # 5. Authentication edge cases. + c.call('GET', '/api/v1/users/me', 401, + headers={'X-User-Id': str(uuid.uuid4())}) + c.call('GET', '/api/v1/users/me', 401) + c.call('GET', '/api/v1/users/me', 401, + headers={'X-User-Id': 'not-a-uuid'}) + + # 6. Invalid limits. + c.call('GET', path + '/messages?limit=0', 400, actor=alice) + c.call('GET', path + '/messages?limit=101', 400, actor=alice) + c.call('GET', path + '/messages?limit=abc', 400, actor=alice) + c.call('GET', path + '/messages?limit=', 400, actor=alice) + + # 7. Invalid chat id in path. + c.call('GET', '/api/v1/chats/not-a-uuid', 400, actor=alice) + + # 8. Extra fields are rejected (additionalProperties: false). + c.call('POST', '/api/v1/users', 400, + body={'username': 'extra_' + uuid.uuid4().hex[:6], + 'display_name': 'X', + 'password': 'p'}) + c.call('POST', path + '/messages', 400, actor=alice, + body={'text': 'hi', 'sender_id': alice['id']}) + + print('PASS: student negative tests') + + +if __name__ == '__main__': + run(main) \ No newline at end of file diff --git a/tests/student-tests/test_pagination.py b/tests/student-tests/test_pagination.py new file mode 100644 index 00000000..c491df80 --- /dev/null +++ b/tests/student-tests/test_pagination.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Walk multi-page history, verify no loss/dup/reorder, and limit change.""" +import uuid +from _bootstrap import TESTS_DIR # noqa: F401 + +from client import Client, arguments, check, run, timestamp + + +def main(): + args = arguments('Student pagination tests') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + + # Create 25 messages with different Unicode payloads. + created = [] + for i in range(25): + created.append(c.message(alice, chat, + 'строка-%02d-😀' % i)['json']) + created.sort(key=lambda m: (timestamp(m['created_at']), uuid.UUID(m['id']).int)) + + # Walk with limit=7 + items = [] + cursor = None + pages = 0 + while True: + q = path + '/messages?limit=7' + if cursor: + from urllib.parse import quote + q += '&cursor=' + quote(cursor, safe='') + page = c.call('GET', q, actor=bob, schema='MessagePage')['json'] + items.extend(page['items']) + pages += 1 + if page['next_cursor'] is None: + break + cursor = page['next_cursor'] + check(pages < 20, 'too many pages') + check(items == created, + 'pagination lost/duplicated/reordered messages') + check(pages == 4, 'expected 4 pages, got %d' % pages) + + # Change limit on continuation + first = c.call('GET', path + '/messages?limit=3', actor=alice, + schema='MessagePage')['json'] + from urllib.parse import quote + second = c.call('GET', + path + '/messages?limit=10&cursor=' + quote(first['next_cursor'], safe=''), + actor=alice, schema='MessagePage')['json'] + combined = first['items'] + second['items'] + check(combined == created[:13], 'limit change broke the cursor') + + # Cursor from another chat must be rejected with 400 + other = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Second', 'member_ids': [alice['id']]}, + schema='Chat')['json'] + c.call('GET', '/api/v1/chats/%s/messages?limit=1' % other['id'], + actor=alice, schema='MessagePage') + # Take a cursor from the original chat and try it on `other` + fake = c.call('GET', path + '/messages?limit=1', actor=alice, + schema='MessagePage')['json']['next_cursor'] + c.call('GET', + '/api/v1/chats/%s/messages?limit=1&cursor=%s' % (other['id'], fake), + 400, actor=alice) + + print('PASS: student pagination tests (4 pages, no loss, cursor scoped)') + + +if __name__ == '__main__': + run(main) \ No newline at end of file diff --git a/tests/student-tests/test_sigterm.py b/tests/student-tests/test_sigterm.py new file mode 100644 index 00000000..30f53a5b --- /dev/null +++ b/tests/student-tests/test_sigterm.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""SIGTERM must stop the container in <= 10s and refuse new requests.""" +import argparse +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request + + +def _http_ok(url): + try: + with urllib.request.urlopen(url, timeout=2) as resp: + return 200 <= resp.status < 300 + except Exception: # noqa: BLE001 + return False + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--container', default=os.environ.get('CONTAINER', 'mtusi-messenger-lab1')) + ap.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080')) + args = ap.parse_args() + + if not _http_ok(args.base_url + '/health/live'): + print('FAIL: server is not alive before SIGTERM') + return 1 + + start = time.monotonic() + subprocess.run(['docker', 'kill', '--signal', 'SIGTERM', args.container], + check=True) + deadline = start + 12 + while time.monotonic() < deadline: + state = subprocess.run( + ['docker', 'inspect', '-f', '{{.State.Running}}', args.container], + capture_output=True, text=True).stdout.strip() + if state == 'false': + break + time.sleep(0.15) + else: + print('FAIL: container did not stop within 12 s') + return 1 + + elapsed = time.monotonic() - start + if elapsed > 10.0: + print('FAIL: SIGTERM took %.2f s (>10)' % elapsed) + return 1 + if _http_ok(args.base_url + '/health/live'): + print('FAIL: server still accepts requests after SIGTERM') + return 1 + print('PASS: SIGTERM stopped container in %.2f s; port refused afterwards' + % elapsed) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/tests/test_persistence.py b/tests/test_persistence.py new file mode 100644 index 00000000..490ceda4 --- /dev/null +++ b/tests/test_persistence.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Verify that user, session and message survive a full service restart.""" +import os +import subprocess +import sys +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +import urllib.request +import urllib.error +import json +import base64 + + +def http(method, url, body=None, headers=None, expect=None): + h = {"Accept": "application/json"} + if headers: + h.update(headers) + raw = None + if body is not None: + raw = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=raw, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=10) + except urllib.error.HTTPError as e: + resp = e + data = resp.read() + if expect is not None: + assert resp.status == expect, f"{method} {url}: got {resp.status}, want {expect}" + return resp.status, json.loads(data) if data else None + + +def wait_ready(base, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + s, _ = http("GET", base + "/health/ready") + if s == 200: + return + except Exception: + pass + time.sleep(1) + raise SystemExit("ready timeout") + + +def main(): + base = os.environ.get("BASE_URL", "http://127.0.0.1:8080") + action = os.environ.get("ACTION_SCRIPT") + if not action: + print("FAIL: ACTION_SCRIPT not set") + sys.exit(1) + + u = "persist_" + uuid.uuid4().hex[:8] + pwd = "SuperSecretPassword123" + s, user = http("POST", base + "/api/v1/users", + body={"username": u, "display_name": "P", "password": pwd}, + expect=201) + alice_id = user["id"] + + cred = base64.b64encode(f"{u}:{pwd}".encode()).decode() + s, sess = http("POST", base + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + cred}, expect=201) + token = sess["token"] + + s, chat = http("POST", base + "/api/v1/chats", + body={"title": "P", "member_ids": [alice_id]}, + headers={"Authorization": "Bearer " + token}, expect=201) + s, msg = http("POST", base + f"/api/v1/chats/{chat['id']}/messages", + body={"text": "persist"}, + headers={"Authorization": "Bearer " + token}, expect=201) + + print(f"created user={alice_id} msg={msg['id']}", file=sys.stderr) + print(f"running action: {action}", file=sys.stderr) + subprocess.run(action, shell=True, check=True) + + wait_ready(base) + + s, me = http("GET", base + "/api/v1/users/me", + headers={"Authorization": "Bearer " + token}, expect=200) + assert me["id"] == alice_id, "user id changed" + + s, hist = http("GET", base + f"/api/v1/chats/{chat['id']}/messages?limit=10", + headers={"Authorization": "Bearer " + token}, expect=200) + assert any(m["id"] == msg["id"] for m in hist["items"]), "message lost" + + print("PASS: persistence verified") + + +if __name__ == "__main__": + main() \ No newline at end of file -- 2.54.0