diff --git a/.env.example b/.env.example index d99d26a2..519a85a7 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,19 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 +POSTGRES_USER=messenger +POSTGRES_PASSWORD=messenger_password +POSTGRES_DB=messenger_db +DATABASE_URL=postgresql+asyncpg://messenger:messenger_password@db:5432/messenger_db +REDIS_URL=redis://redis:6379/0 +SESSION_STORE=redis IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY_ID=api-user +S3_SECRET_ACCESS_KEY=api-user-secret-2026 MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 +MAX_IMAGE_PIXELS=40000000 \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..b7fb1106 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,72 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Долда Кирилл / БПИ2301 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: Трек 2 -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +**Что реализовано и что перенесено:** -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Из lab3 перенесены PostgreSQL, Redis, Alembic, Bearer-токены, реплики, Idempotency-Key, лимиты CPU/RAM, X-Instance-Id. + +Добавлены MinIO с приватным bucket, таблица attachments, поле attachment_ids в сообщениях, загрузка и скачивание файлов через API, проверки MIME, пикселей и размера, S3-пользователь api-user с ограниченными правами (не admin), reconciler для orphan и missing, тесты вложений, RAM-эксперимент, stream-limit, тесты grace period и используемых вложений. + +**Команды запуска из чистого clone:** + +**1. Подготовка** +cp .env.example .env + +**2. Сборка и запуск** +docker compose up -d --build +docker compose restart nginx + +**3. Ожидание готовности** +until curl -sf http://localhost:8080/health/ready + +**4. Миграции (однократно, до трафика)** +docker compose exec -w /app app1 alembic upgrade head + +**Bootstrap S3 проверка:** +docker compose exec minio sh -c 'mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc admin user list myminio' + +Ожидаемо: api-user, minioadmin + +**5. Остановка контейнера** +docker compose down + +**Результаты проверок и ссылка на REPORT.md:** + +- make test — PASS: lab4 public API scenario. + +- make test-attachments — PASS (5/5) + +- make stream-limit — PASS: 100 МБ -> 413 за 0.10s, память не растёт. + +- make ram-experiment — 10/10 (201), avg 6.37 МБ, пик памяти app1 ~290 МБ / 512 МБ, OOM нет. + +- make reconcile-test — PASS: [ORPHAN] + [MISSING] в dry-run -> [DELETED] в execute -> повторный dry-run 0/0 (идемпотентность). + +- make grace-period-test — PASS: [SKIP] (1 сек < grace 10) -> [ORPHAN] (12 сек > grace 10) -> [DELETED]. + +- make used-attachment-test — PASS + +- make reconcile-cost — 50 файлов / 1.012s (~2 мс/файл, линейная O(N)). + +- make private-bucket — 403 AccessDenied (bucket приватный). + +- make s3-user-check — api-user не admin: mc admin user add / mc admin policy attach / mc admin user remove -> Access Denied. + +**Ограничения / известные ошибки:** + +- Grace period в тестах — 10 секунд (в проде — 1 час: защита от удаления свежих загрузок, которые могут быть в процессе). +- Стоимость reconciler — 50 файлов / 1.012s (~2 мс/файл). + +**Источники и использование ИИ:** + +FastAPI, SQLAlchemy, asyncpg, Alembic, passlib[bcrypt], redis, psycopg2-binary, minio, filetype, Pillow, python-multipart, pytest, nginx:alpine, postgres:16-alpine, redis:7-alpine, pgsty/silo:latest + +- [+] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [+] Контракт и публичные тесты не ослаблены. +- [+] make check и make test выполнены; приложен реальный вывод. +- [+] Сценарии отказов/безопасности своего уровня воспроизведены. +- [+] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [+] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2afecc48 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.13-slim + +WORKDIR /app + +COPY requirements-dev.txt . +RUN pip install --no-cache-dir -r requirements-dev.txt + +RUN useradd --create-home --shell /bin/bash appuser + +COPY app/ app/ + +COPY alembic.ini . +COPY alembic/ alembic/ +RUN chown -R appuser:appuser /app/alembic +COPY scripts/ /app/scripts/ + +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/live')" || exit 1 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "10", "--reload"] + diff --git a/Makefile b/Makefile index 5d86d171..5abc715e 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -25,6 +25,138 @@ test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) -test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) - $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) +explain-10k: + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM messages WHERE chat_id = '22222222-2222-2222-2222-222222222222' ORDER BY created_at, id LIMIT 50 OFFSET 0;" + +explain-composite: + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM messages WHERE chat_id = '66666666-6666-4666-8666-666666666666' ORDER BY created_at, id LIMIT 50 OFFSET 0;" | tee evidence/explain_composite.txt + +explain-simple: + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "DROP INDEX IF EXISTS ix_messages_chat_created_id;" + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "CREATE INDEX ix_messages_chat_only ON messages (chat_id); ANALYZE messages;" + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM messages WHERE chat_id = '66666666-6666-4666-8666-666666666666' ORDER BY created_at, id LIMIT 50 OFFSET 0;" | tee evidence/explain_simple.txt + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "DROP INDEX ix_messages_chat_only;" + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -c "CREATE INDEX ix_messages_chat_created_id ON messages (chat_id, created_at, id);" + +backup-restore: + @echo "Делаем бекап pg_dump" + time docker exec -it mtusi_messenger_db pg_dump -U messenger -d messenger_db > evidence/backup.sql + @echo "Создаем новую базу messenger_restore" + docker exec -it mtusi_messenger_db psql -U messenger -d postgres -c "DROP DATABASE IF EXISTS messenger_restore;" + docker exec -it mtusi_messenger_db psql -U messenger -d postgres -c "CREATE DATABASE messenger_restore;" + @echo "Копируем бекап в контейнер и восстанавливаем" + docker cp evidence/backup.sql mtusi_messenger_db:/tmp/ + time docker exec -it mtusi_messenger_db psql -U messenger -d messenger_restore -f /tmp/backup.sql > /dev/null + @echo "В основной базе messenger_db:" + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_db -t -c "SELECT count(*) FROM messages;" + @echo "В восстановленной базе messenger_restore:" + docker exec -it mtusi_messenger_db psql -U messenger -d messenger_restore -t -c "SELECT count(*) FROM messages;" + +test-race: + @echo "Запускаем два одновременных запроса на регистрацию" + @RACE_USER="race_$$(date +%s)"; \ + curl -s -o /dev/null -w "Запрос 1: %{http_code}\n" -X POST http://localhost:8080/api/v1/users -H "Content-Type: application/json" -d "{\"username\": \"$$RACE_USER\", \"display_name\": \"Race\", \"password\": \"secret1234567\"}" & \ + curl -s -o /dev/null -w "Запрос 2: %{http_code}\n" -X POST http://localhost:8080/api/v1/users -H "Content-Type: application/json" -d "{\"username\": \"$$RACE_USER\", \"display_name\": \"Race\", \"password\": \"secret1234567\"}" & \ + wait + +test-redis: + @echo "Ставим Redis на паузу" + docker pause mtusi_messenger_redis + @echo "Запрос" + time curl -s -w "\nHTTP Статус: %{http_code}\n" http://localhost:8080/api/v1/users/me -H "Authorization: Bearer test_token" + @echo "Снимаем паузу" + docker unpause mtusi_messenger_redis + +test-health-db: + @echo "Останавливаем БД" + docker compose stop db + @echo "Проверяем health ready" + curl -s -o /dev/null -w "HTTP Статус: %{http_code}\n" http://localhost:8080/health/ready + docker compose start db + +test-health-redis: + @echo "Останавливаем" + docker compose stop redis + @echo "Проверяем health ожидая 503 " + curl -s -o /dev/null -w "HTTP Статус: %{http_code}\n" http://localhost:8080/health/ready + docker compose start redis + +test-dependencies: + $(PYTHON) -m pytest tests/test_resilience.py -v + +failover: + ./scripts/failover-test.sh app1 + +failover-app2: + ./scripts/failover-test.sh app2 + +rolling-load: + ./scripts/rolling-load.sh + +rolling-update: + ./scripts/rolling-update.sh + +test-lab3: + $(PYTHON) -m pytest tests/test_lab3.py -v + +test-attachments: + $(PYTHON) -m pytest tests/test_attachments.py -v + +reconcile-dry: + docker compose exec -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run + +reconcile-execute: + docker compose exec -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute + +reconcile-test: + ./scripts/run-reconciler-test.sh + +ram-experiment: + @echo "В другом терминале для определения нагрузки: docker stats --format 'table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}'" + mkdir -p evidence + docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" > evidence/ram-experiment.txt + docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/ram-experiment.py 2>&1 | tee -a evidence/ram-experiment.txt + docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" >> evidence/ram-experiment.txt + +stream-limit: + mkdir -p evidence + @echo "=== ДО ===" > evidence/stream-limit.txt + @docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" >> evidence/stream-limit.txt + @echo "" >> evidence/stream-limit.txt + docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/test-stream-limit.py 2>&1 | tee -a evidence/stream-limit.txt + @echo "" >> evidence/stream-limit.txt + @echo "=== ПОСЛЕ ===" >> evidence/stream-limit.txt + @docker stats --no-stream --format "table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}" >> evidence/stream-limit.txt + +private-bucket: + mkdir -p evidence + @OBJECT_KEY=$$(docker compose exec -T db psql -U messenger -d messenger_db -t -c "SELECT object_key FROM attachments ORDER BY created_at DESC LIMIT 1;" | tr -d ' \n'); \ + echo "Object key: $$OBJECT_KEY"; \ + { \ + echo "=== Проверка private bucket ==="; \ + echo "Object key: $$OBJECT_KEY"; \ + echo ""; \ + echo "=== Прямой GET без авторизации (ожидаем 403) ==="; \ + curl -i "http://localhost:9000/messenger-attachments/$$OBJECT_KEY"; \ + } > evidence/private-bucket.txt 2>&1; \ + cat evidence/private-bucket.txt + +s3-user-check: + @echo "=== 1. api-user пытается создать пользователя (admin) ===" + @docker compose exec -T minio sh -c 'mc alias set myminio http://localhost:9000 api-user api-user-secret-2026 > /dev/null && mc admin user add myminio hacker hacker123 2>&1' | head -3 + @echo "" + @echo "=== 2. api-user пытается изменить политику (admin) ===" + @docker compose exec -T minio sh -c 'mc alias set myminio http://localhost:9000 api-user api-user-secret-2026 > /dev/null && mc admin policy attach myminio readwrite --user hacker 2>&1' | head -3 + @echo "" + @echo "=== 3. api-user пытается удалить пользователя (admin) ===" + @docker compose exec -T minio sh -c 'mc alias set myminio http://localhost:9000 api-user api-user-secret-2026 > /dev/null && mc admin user remove myminio minioadmin 2>&1' | head -3 + +reconcile-cost: + @echo "=== Стоимость обхода reconciler ===" + time docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run 2>&1 | tee evidence/track2-cost-50.txt + +grace-period-test: + ./scripts/run-grace-period-test.sh + +used-attachment-test: + ./scripts/run-used-attachment-test.sh \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..aab03152 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,143 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Долда Кирилл, БПИ2301 +- Целевая оценка; 5, Трек 2 (Согласованность двух хранилищ) - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники, библиотеки и помощь ИИ: FastAPI, SQLAlchemy, asyncpg, Alembic, passlib[bcrypt], redis, psycopg2-binary, minio, filetype, Pillow, python-multipart. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования: Docker Desktop, Python 3.13 (для локальных тестов). + +Версии runtime/образов: +python:3.13-slim +postgres:16-alpine +redis:7-alpine +nginx:alpine +pgsty/silo:latest +pgsty/mc:latest + +Python-пакеты: fastapi==0.115.0, uvicorn[standard]==0.52.4, pydantic==2.13.5, sqlalchemy[asyncio]==2.1.0, asyncpg==0.31.0, redis==6.2.0, passlib==1.7.4, bcrypt==4.1.2, alembic==1.14.0, psycopg2-binary, pytest==8.3.4, python-multipart==0.0.9, minio==7.2.9, filetype==1.2.0, Pillow==10.4.0. + +Создание локальных secrets/.env: +cp .env.example .env + +Сборка и запуск: +docker compose up -d --build +docker compose restart nginx + +Ожидание готовности: +until curl -sf http://localhost:8080/health/ready + +Миграции (однократно, до запуска трафика): +docker compose exec -w /app app1 alembic upgrade head + +Bootstrap S3: +**minio-init автоматически: создаёт бакет messenger-attachments и создает api-user** +**Проверить:** +docker compose exec minio sh -c 'mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc admin user list myminio' +**Ожидаемо: api-user, minioadmin** + +Тесты: + +make check - offline-проверка структуры и синтаксиса +make test - публичный smoke-тест контракта API (PASS) +make test-attachments - тесты вложений (5 сценариев, 5 passed) +make test-lab3 - тесты лабы 3 (идемпотентность + общая сессия) +make test-security - тесты безопасности (пароль, logout, expiry) +make test-dependencies - тесты отказов зависимостей (БД -> 503, Redis -> 500) +make failover / failover-app2 - остановка реплики -> сервис жив +make rolling-load / rolling-update - rolling update под нагрузкой +make ram-experiment - RAM-эксперимент (10 одновременных загрузок) +make stream-limit - потоковый лимит (100 МБ -> 413) +make reconcile-dry / reconcile-execute / reconcile-test - reconciler +make grace-period-test - grace period (свежие не удаляются) +make used-attachment-test - используемые не удаляются +make reconcile-cost - стоимость обхода reconciler +make private-bucket - приватный bucket (403 AccessDenied) +make s3-user-check - api-user не admin (Access Denied x3) +docker stats - CPU/RAM при нагрузке + +Безопасная остановка (данные в томах сохраняются): +docker compose down + +Намеренное удаление учебных данных: +docker compose down -v ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Схема процессов/хранилищ: +Клиент -> Nginx (LB) -> API * 2 (app1, app2) -> PostgreSQL (метаданные) + MinIO (байты) + Redis (сессии). + +Источник истины: +- PostgreSQL: пользователи, чаты, участники, сообщения, метаданные вложений. +- MinIO: байты файлов (приватный бакет messenger-attachments, object_key — UUID). + +Транзакционные границы: +- Reconciler: orphan — удалить из MinIO (если старше grace period); missing — удалить из БД (если не используется привязан к таблице в БД). + +Лимиты: +- MAX_SIZE = 10 МБ на файл (потоковое чтение request.stream() чанками, проверка на каждом чанке). +- MAX_PIXELS = 40 млн пикселей +- memory: 512M на app1/app2 (docker-compose.yml). +- Nginx: client_max_body_size 10M + +Что перенесено из lab3: PostgreSQL, Redis для сессий, миграции, Bearer-токены, реплики app1/app2, Nginx (least_conn), X-Instance-Id, Idempotency-Key, лимиты CPU/RAM, failover-тесты, rolling update. + +Что изменено: добавлены MinIO (форк pgsty/silo), таблица attachments (метаданные), приватный bucket (mc anonymous set none), reconciler (scripts/reconcile.py), тесты вложений (tests/test_attachments.py), RAM-эксперимент (scripts/ram-experiment.py), grace period (scripts/run-grace-period-test.sh). + +Соответствие .env.example: +- S3_ENDPOINT=http://minio:9000 — адрес MinIO в Docker-сети. +- S3_BUCKET=messenger-attachments — бакет (приватный). +- S3_ACCESS_KEY_ID=api-user / S3_SECRET_ACCESS_KEY=api-user-secret-2026 — API-пользователь (не root). +- MAX_UPLOAD_BYTES=10485760 (10 МБ) / MAX_IMAGE_PIXELS=40000000 (40 млн). ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +|------------------------------------|---------------------------|----------------------------------------------------------------------------| --- | +| Публичный smoke | make test | PASS: lab4 public API scenario | — | +| Тесты вложений | make test-attachments | 5 passed | evidence/test-attachments.txt | +| Приватный bucket | make private-bucket | GET напрямую → 403 AccessDenied | evidence/private-bucket.txt | +| S3 credentials (api-user не admin) | make s3-user-check | evidence/s3-credentials.txt | +| RAM-эксперимент (10 загрузок) | make ram-experiment | 10/10 (201), avg 6.37 МБ, пик памяти app1 ~290 МБ / 512 МБ, OOM нет | evidence/ram-experiment.txt | +| Потоковый лимит (100 МБ -> 413) | make stream-limit | Status 413 за 0.11s, память не растёт (потоковое чтение прервало на 10 МБ) | evidence/stream-limit.txt | +| Reconciler: обнаружение | make reconcile-test | [ORPHAN] + [MISSING] в dry-run, [DELETED] в execute, повторный dry-run 0/0 | evidence/reconciler-full.txt | +| Grace period (свежие не удаляются) | make grace-period-test | [SKIP] (1 сек) → [ORPHAN] (12 сек) → [DELETED] | evidence/track2-grace-period.txt | +| Используемые не удаляются | make used-attachment-test | [SKIP] для missing с messages.attachment_ids, COUNT=1 → PASS | evidence/track2-used-attachment.txt | +| Стоимость обхода | make reconcile-cost | 50 файлов за 1.012s | evidence/track2-cost-50.txt | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### RAM-эксперимент + +Гипотеза: лимиты (10 МБ на файл) не дают одной загрузке занять неограниченную RAM. 10 одновременных загрузок не превысят лимит 512 МБ. + +Методика: scripts/ram-experiment.py — 10 одновременных загрузок через, файлы ~5-7 МБ. Параллельно docker stats. + +Результат: 10/10 (201), avg 6.37 МБ, суммарно 63.7 МБ, общее время ~1.6s (параллельно). Пик памяти app1 ~290 МБ / 512 МБ (57% лимита), app2 ~94 МБ (Nginx распределил). После нагрузки память вернулась к ~97 МБ. OOM нет. + +Вывод: лимиты работают — даже 10 загрузок по 6 МБ не выходят за 512 МБ. Потоковое чтение ограничивает пик ≤ 10 МБ на одну загрузку. + +### Трек 2: Согласованность двух хранилищ + +Гипотеза: при сбое между записью в MinIO и записью в PostgreSQL появляются расхождения: orphan (файл в S3 без БД), missing (запись в БД без S3). Reconciler находит оба и безопасно исправляет, не удаляя свежие (grace period) и используемые. + +Результаты: + +| Сценарий | Наблюдаемо | Файл | +| --- | --- | --- | +| Обнаружение (dry-run) | [ORPHAN] + [MISSING] | evidence/reconciler-full.txt | +| Исправление (execute) | [DELETED] + COUNT=0 | evidence/reconciler-full.txt | +| Идемпотентность | повторный dry-run → 0/0 | evidence/reconciler-full.txt | +| Grace period | [SKIP] (1 сек) → [ORPHAN] (12 сек) → [DELETED] | evidence/track2-grace-period.txt | +| Используемые | [SKIP] + COUNT=1 → PASS | evidence/track2-used-attachment.txt | +| Стоимость обхода | 50 файлов / 1.012s (~2 мс/файл) | evidence/track2-cost-50.txt | -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Что выдерживается: отказ процесса (реплики), rolling update, общая сессия, идемпотентность, миграции однократно, приватный bucket, S3 credentials (api-user не admin), потоковое чтение (100 МБ → 413 без OOM), reconciler (orphan + missing), grace period. + +Подтверждённое улучшение (трек 2): reconciler восстанавливает согласованность MinIO/PostgreSQL — находит orphan и missing, идемпотентен, защищает свежие (grace period) и используемые вложения. Стоимость ~2 мс/файл (50 файлов / 1.012s). + diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..0e50bdb1 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,117 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +# Use forward slashes (/) also on windows to provide an os agnostic path +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +# version_path_separator = newline +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 00000000..79b1a8dd --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,91 @@ +from app.models.base import Base +from app.models.user import User + +from app.models.chat import Chat, ChatMember +from app.models.message import Message +from app.config import DATABASE_URL + +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +from app.models.attachment import Attachment + + + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config +config.set_main_option("sqlalchemy.url", DATABASE_URL.replace("+asyncpg", "+psycopg2")) + + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 00000000..fbc4b07d --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/08fe661df060_create_tables.py b/alembic/versions/08fe661df060_create_tables.py new file mode 100644 index 00000000..51c3cf25 --- /dev/null +++ b/alembic/versions/08fe661df060_create_tables.py @@ -0,0 +1,30 @@ +"""create tables + +Revision ID: 08fe661df060 +Revises: bc35c1b6b748 +Create Date: 2026-10-31 11:52:26.342048 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '08fe661df060' +down_revision: Union[str, None] = 'bc35c1b6b748' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/alembic/versions/8203fbed8a46_add_timestamps_to_attachments.py b/alembic/versions/8203fbed8a46_add_timestamps_to_attachments.py new file mode 100644 index 00000000..d7e3a2d7 --- /dev/null +++ b/alembic/versions/8203fbed8a46_add_timestamps_to_attachments.py @@ -0,0 +1,31 @@ +"""add timestamps to attachments + +Revision ID: 8203fbed8a46 +Revises: b0f31212df2c +Create Date: 2026-11-11 15:59:23.045039 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '8203fbed8a46' +down_revision: Union[str, None] = 'b0f31212df2c' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Добавляем колонки created_at и updated_at + op.add_column('attachments', sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False)) + op.add_column('attachments', sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False)) + # УДАЛЕНО: op.alter_column('attachments', 'user_id', ...) - чтобы не упасть на старых записях + + +def downgrade() -> None: + op.drop_column('attachments', 'updated_at') + op.drop_column('attachments', 'created_at') + # УДАЛЕНО: op.alter_column('attachments', 'user_id', nullable=True) \ No newline at end of file diff --git a/alembic/versions/b0f31212df2c_add_owner_and_sha256_to_attachments.py b/alembic/versions/b0f31212df2c_add_owner_and_sha256_to_attachments.py new file mode 100644 index 00000000..c095702f --- /dev/null +++ b/alembic/versions/b0f31212df2c_add_owner_and_sha256_to_attachments.py @@ -0,0 +1,35 @@ +"""add owner and sha256 to attachments + +Revision ID: b0f31212df2c +Revises: d78a7aaedac6 +Create Date: 2026-11-11 15:39:07.391800 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'b0f31212df2c' +down_revision: Union[str, None] = 'd78a7aaedac6' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # 1. Добавляем user_id с nullable=True, чтобы не упасть на старых данных + op.add_column('attachments', sa.Column('user_id', sa.UUID(), nullable=True)) + + # 2. Добавляем sha256 + op.add_column('attachments', sa.Column('sha256', sa.String(), nullable=True)) + + # 3. Создаем внешний ключ + op.create_foreign_key(None, 'attachments', 'users', ['user_id'], ['id'], ondelete='CASCADE') + + +def downgrade() -> None: + op.drop_constraint(None, 'attachments', type_='foreignkey') + op.drop_column('attachments', 'sha256') + op.drop_column('attachments', 'user_id') \ No newline at end of file diff --git a/alembic/versions/b24373e3b620_create_tables.py b/alembic/versions/b24373e3b620_create_tables.py new file mode 100644 index 00000000..eac7789d --- /dev/null +++ b/alembic/versions/b24373e3b620_create_tables.py @@ -0,0 +1,67 @@ +"""create tables + +Revision ID: b24373e3b620 +Revises: +Create Date: 2026-10-24 06:54:22.471827 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'b24373e3b620' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('chats', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('title', sa.String(length=100), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('users', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('username', sa.String(length=32), nullable=False), + sa.Column('display_name', sa.String(length=100), nullable=False), + sa.Column('password_hash', sa.String(length=255), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_users_username'), 'users', ['username'], unique=True) + op.create_table('chat_members', + sa.Column('chat_id', sa.UUID(), nullable=False), + sa.Column('user_id', sa.UUID(), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('chat_id', 'user_id') + ) + op.create_table('messages', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('chat_id', sa.UUID(), nullable=False), + sa.Column('sender_id', sa.UUID(), nullable=False), + sa.Column('text', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['sender_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('ix_messages_chat_created_id', 'messages', ['chat_id', 'created_at', 'id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('ix_messages_chat_created_id', table_name='messages') + op.drop_table('messages') + op.drop_table('chat_members') + op.drop_index(op.f('ix_users_username'), table_name='users') + op.drop_table('users') + op.drop_table('chats') + # ### end Alembic commands ### diff --git a/alembic/versions/bc35c1b6b748_seed_explain_test_10k.py b/alembic/versions/bc35c1b6b748_seed_explain_test_10k.py new file mode 100644 index 00000000..19897503 --- /dev/null +++ b/alembic/versions/bc35c1b6b748_seed_explain_test_10k.py @@ -0,0 +1,43 @@ +"""seed explain test 10k + +Revision ID: bc35c1b6b748 +Revises: f9b85be2b710 +Create Date: 2026-10-24 07:04:09.294404 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'bc35c1b6b748' +down_revision: Union[str, None] = 'f9b85be2b710' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute(""" + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ('11111111-1111-1111-1111-111111111111', 'explain_test', 'Test', 'hash', NOW()) + ON CONFLICT DO NOTHING; + """) + + op.execute(""" + INSERT INTO chats (id, title, created_at) + VALUES ('22222222-2222-2222-2222-222222222222', 'Explain Chat', NOW()) + ON CONFLICT DO NOTHING; + """) + + op.execute(""" + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), '22222222-2222-2222-2222-222222222222', '11111111-1111-1111-1111-111111111111', 'Message ' || i, NOW() - (i || ' minutes')::interval + FROM generate_series(1, 10000) AS i; + """) + +def downgrade() -> None: + op.execute("DELETE FROM messages WHERE chat_id = '22222222-2222-2222-2222-222222222222';") + op.execute("DELETE FROM chats WHERE id = '22222222-2222-2222-2222-222222222222';") + op.execute("DELETE FROM users WHERE id = '11111111-1111-1111-1111-111111111111';") \ No newline at end of file diff --git a/alembic/versions/d78a7aaedac6_add_attachment_ids_to_messages.py b/alembic/versions/d78a7aaedac6_add_attachment_ids_to_messages.py new file mode 100644 index 00000000..6e284653 --- /dev/null +++ b/alembic/versions/d78a7aaedac6_add_attachment_ids_to_messages.py @@ -0,0 +1,30 @@ +"""add attachment_ids to messages + +Revision ID: d78a7aaedac6 +Revises: ea4f73899364 +Create Date: 2026-11-11 15:16:50.773392 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'd78a7aaedac6' +down_revision: Union[str, None] = 'ea4f73899364' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column('messages', sa.Column('attachment_ids', sa.JSON(), nullable=True)) + + op.execute("UPDATE messages SET attachment_ids = '[]'::json WHERE attachment_ids IS NULL") + + op.alter_column('messages', 'attachment_ids', nullable=False, server_default='[]') + + +def downgrade() -> None: + op.drop_column('messages', 'attachment_ids') \ No newline at end of file diff --git a/alembic/versions/ea4f73899364_add_attachments_table.py b/alembic/versions/ea4f73899364_add_attachments_table.py new file mode 100644 index 00000000..ffb86529 --- /dev/null +++ b/alembic/versions/ea4f73899364_add_attachments_table.py @@ -0,0 +1,40 @@ +"""add attachments table + +Revision ID: ea4f73899364 +Revises: 08fe661df060 +Create Date: 2026-11-11 14:37:24.096041 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'ea4f73899364' +down_revision: Union[str, None] = '08fe661df060' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('attachments', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('object_key', sa.String(), nullable=False), + sa.Column('filename', sa.String(), nullable=False), + sa.Column('mime_type', sa.String(), nullable=False), + sa.Column('size_bytes', sa.Integer(), nullable=False), + sa.Column('chat_id', sa.UUID(), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('object_key') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('attachments') + # ### end Alembic commands ### diff --git a/alembic/versions/f9b85be2b710_seed_load_test_data.py b/alembic/versions/f9b85be2b710_seed_load_test_data.py new file mode 100644 index 00000000..35e4be61 --- /dev/null +++ b/alembic/versions/f9b85be2b710_seed_load_test_data.py @@ -0,0 +1,69 @@ +"""seed load test data + +Revision ID: f9b85be2b710 +Revises: b24373e3b620 +Create Date: ... +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'f9b85be2b710' +down_revision: Union[str, None] = 'b24373e3b620' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute(""" + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ('55555555-5555-4555-8555-555555555555', 'load_test', 'Load', 'hash', NOW()) + ON CONFLICT DO NOTHING; + """) + + op.execute(""" + INSERT INTO chats (id, title, created_at) VALUES + ('66666666-6666-4666-8666-666666666666', 'Big Chat', NOW()), + ('77777777-7777-4777-8777-777777777777', 'Small Chat 1', NOW()), + ('88888888-8888-4888-8888-888888888888', 'Small Chat 2', NOW()) + ON CONFLICT DO NOTHING; + """) + + op.execute(""" + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), '66666666-6666-4666-8666-666666666666', '55555555-5555-4555-8555-555555555555', 'Big ' || i, NOW() - (i || ' minutes')::interval + FROM generate_series(1, 80000) AS i; + """) + + op.execute(""" + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), '77777777-7777-4777-8777-777777777777', '55555555-5555-4555-8555-555555555555', 'Small1 ' || i, NOW() - (i || ' minutes')::interval + FROM generate_series(1, 10000) AS i; + """) + + op.execute(""" + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), '88888888-8888-4888-8888-888888888888', '55555555-5555-4555-8555-555555555555', 'Small2 ' || i, NOW() - (i || ' minutes')::interval + FROM generate_series(1, 10000) AS i; + """) + + +def downgrade() -> None: + op.execute(""" + DELETE FROM messages WHERE chat_id IN ( + '66666666-6666-4666-8666-666666666666', + '77777777-7777-4777-8777-777777777777', + '88888888-8888-4888-8888-888888888888' + ); + """) + op.execute(""" + DELETE FROM chats WHERE id IN ( + '66666666-6666-4666-8666-666666666666', + '77777777-7777-4777-8777-777777777777', + '88888888-8888-4888-8888-888888888888' + ); + """) + op.execute("DELETE FROM users WHERE id = '55555555-5555-4555-8555-555555555555';") \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..7c029933 --- /dev/null +++ b/app/config.py @@ -0,0 +1,28 @@ +import os + +DATABASE_URL = os.getenv( + "DATABASE_URL", + "postgresql+asyncpg://messenger:messenger_password@db:5432/messenger_db", +) +REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) + +CODE_MAP = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + 500: "internal_error", +} + +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://minio:9000") +S3_REGION = os.getenv("S3_REGION", "us-east-1") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = os.getenv("S3_ACCESS_KEY_ID", "minioadmin") +S3_SECRET_ACCESS_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "minioadmin") +MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) \ No newline at end of file diff --git a/app/database.py b/app/database.py new file mode 100644 index 00000000..b0447a18 --- /dev/null +++ b/app/database.py @@ -0,0 +1,13 @@ +from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker +from app.config import DATABASE_URL + +engine = create_async_engine( + DATABASE_URL, + pool_size=5, + max_overflow=10, + pool_timeout=30, + pool_recycle=1800, + echo=False, +) + +async_session_maker = async_sessionmaker(engine, expire_on_commit=False) \ No newline at end of file diff --git a/app/dependencies.py b/app/dependencies.py new file mode 100644 index 00000000..647232bc --- /dev/null +++ b/app/dependencies.py @@ -0,0 +1,6 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from app.database import async_session_maker + +async def get_session(): + async with async_session_maker() as session: + yield session \ No newline at end of file diff --git a/app/exceptions.py b/app/exceptions.py new file mode 100644 index 00000000..0e6b18a2 --- /dev/null +++ b/app/exceptions.py @@ -0,0 +1,37 @@ +from fastapi import HTTPException, Request +from fastapi.responses import JSONResponse +from fastapi.exceptions import RequestValidationError +from app.config import CODE_MAP + + +def error_response(request: Request, status_code: int, code: str, message: str, extra_headers: dict = None): + request_id = getattr(request.state, "request_id", "unknown") + + + headers = {"X-Request-Id": request_id} + if extra_headers: + headers.update(extra_headers) + + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers=headers, + ) + + +async def http_exception_handler(request: Request, exc: HTTPException): + code = CODE_MAP.get(exc.status_code, "internal_error") + return error_response(request, exc.status_code, code, str(exc.detail), extra_headers=exc.headers) + + +async def validation_exception_handler(request: Request, exc: RequestValidationError): + return error_response(request, 400, "invalid_request", "Ошибка валидации данных") + +async def internal_error_handler(request: Request, exc: Exception): + return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера") \ No newline at end of file diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..14277d99 --- /dev/null +++ b/app/main.py @@ -0,0 +1,31 @@ +from fastapi import FastAPI +from fastapi.exceptions import RequestValidationError +from fastapi import HTTPException +import uuid +from app.middleware import RequestIdMiddleware +from app.exceptions import http_exception_handler, validation_exception_handler +from app.routers import health, users, chats, messages, auth, attachments + + +app = FastAPI( + title="MTUSI Messenger", + version="1.1.0", +) + +INSTANCE_ID = str(uuid.uuid4()) +@app.middleware("http") +async def add_instance_id_header(request, call_next): + response = await call_next(request) + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + +app.add_middleware(RequestIdMiddleware) +app.add_exception_handler(HTTPException, http_exception_handler) +app.add_exception_handler(RequestValidationError, validation_exception_handler) +app.include_router(health.router) +app.include_router(users.router) +app.include_router(chats.router) +app.include_router(messages.router) +app.include_router(auth.router) +app.include_router(attachments.router_upload) +app.include_router(attachments.router_download) \ No newline at end of file diff --git a/app/middleware.py b/app/middleware.py new file mode 100644 index 00000000..142c49e4 --- /dev/null +++ b/app/middleware.py @@ -0,0 +1,11 @@ +import uuid as uuid_lib +from starlette.middleware.base import BaseHTTPMiddleware + + +class RequestIdMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request, call_next): + request_id = str(uuid_lib.uuid4()) + request.state.request_id = request_id + response = await call_next(request) + response.headers["X-Request-Id"] = request_id + return response \ No newline at end of file diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/models/attachment.py b/app/models/attachment.py new file mode 100644 index 00000000..ccbb4394 --- /dev/null +++ b/app/models/attachment.py @@ -0,0 +1,19 @@ +import uuid +from datetime import datetime +from sqlalchemy import String, Integer, ForeignKey, DateTime, func +from sqlalchemy.orm import Mapped, mapped_column +from app.models.base import Base + + +class Attachment(Base): + __tablename__ = "attachments" + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + object_key: Mapped[str] = mapped_column(String, unique=True) + filename: Mapped[str] = mapped_column(String) + mime_type: Mapped[str] = mapped_column(String) + size_bytes: Mapped[int] = mapped_column(Integer) + chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id")) + user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + sha256: Mapped[str] = mapped_column(String, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) \ No newline at end of file diff --git a/app/models/base.py b/app/models/base.py new file mode 100644 index 00000000..22784168 --- /dev/null +++ b/app/models/base.py @@ -0,0 +1,4 @@ +from sqlalchemy.orm import DeclarativeBase + +class Base(DeclarativeBase): + pass \ No newline at end of file diff --git a/app/models/chat.py b/app/models/chat.py new file mode 100644 index 00000000..7bfed508 --- /dev/null +++ b/app/models/chat.py @@ -0,0 +1,35 @@ +import uuid +from datetime import datetime + +from sqlalchemy import String, DateTime, ForeignKey, func +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.dialects.postgresql import UUID + +from app.models.base import Base + + +class Chat(Base): + __tablename__ = "chats" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + title: Mapped[str] = mapped_column(String(100), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + +class ChatMember(Base): + __tablename__ = "chat_members" + + chat_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("chats.id", ondelete="CASCADE"), + primary_key=True, + ) + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + primary_key=True, + ) \ No newline at end of file diff --git a/app/models/message.py b/app/models/message.py new file mode 100644 index 00000000..291e5668 --- /dev/null +++ b/app/models/message.py @@ -0,0 +1,36 @@ +import uuid +from datetime import datetime + +from sqlalchemy import Text, DateTime, ForeignKey, Index, func, JSON +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.dialects.postgresql import UUID + +from app.models.base import Base + + +class Message(Base): + __tablename__ = "messages" + + attachment_ids: Mapped[list] = mapped_column(JSON, default=list) + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + chat_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("chats.id", ondelete="CASCADE"), + nullable=False, + ) + sender_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ) + text: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + __table_args__ = ( + Index("ix_messages_chat_created_id", "chat_id", "created_at", "id"), + ) \ No newline at end of file diff --git a/app/models/user.py b/app/models/user.py new file mode 100644 index 00000000..03ab1ace --- /dev/null +++ b/app/models/user.py @@ -0,0 +1,24 @@ +import uuid +from datetime import datetime + +from sqlalchemy import String, DateTime, func +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.dialects.postgresql import UUID + +from app.models.base import Base + + +class User(Base): + __tablename__ = "users" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + username: Mapped[str] = mapped_column( + String(32), unique=True, nullable=False, index=True + ) + display_name: Mapped[str] = mapped_column(String(100), nullable=False) + password_hash: Mapped[str] = mapped_column(String(255), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) \ No newline at end of file diff --git a/app/pagination.py b/app/pagination.py new file mode 100644 index 00000000..ca387009 --- /dev/null +++ b/app/pagination.py @@ -0,0 +1,20 @@ +def paginate(items: list, limit: int, cursor: str | None): + start = 0 + if cursor is not None: + try: + start = int(cursor) + except ValueError: + start = 0 + + if start < 0: + start = 0 + + page = items[start : start + limit] + next_index = start + limit + + if next_index < len(items): + next_cursor = str(next_index) + else: + next_cursor = None + + return page, next_cursor \ No newline at end of file diff --git a/app/redis_client.py b/app/redis_client.py new file mode 100644 index 00000000..0d6aa009 --- /dev/null +++ b/app/redis_client.py @@ -0,0 +1,9 @@ +import redis.asyncio as redis +from app.config import REDIS_URL + +redis_client = redis.from_url( + REDIS_URL, + decode_responses=True, + socket_timeout=3, + socket_connect_timeout=3, +) \ No newline at end of file diff --git a/app/routers/__init__.py b/app/routers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/routers/attachments.py b/app/routers/attachments.py new file mode 100644 index 00000000..11c34b39 --- /dev/null +++ b/app/routers/attachments.py @@ -0,0 +1,172 @@ +import uuid +import filetype +import io +import hashlib +from datetime import datetime, timezone +from PIL import Image +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi.responses import StreamingResponse +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select + +from app.dependencies import get_session +from app.security import get_current_user +from app.models.user import User +from app.models.attachment import Attachment +from app.models.chat import ChatMember +from app.storage import minio_client, BUCKET + +router_upload = APIRouter(prefix="/api/v1/chats/{chat_id}/attachments", tags=["attachments"]) +router_download = APIRouter(prefix="/api/v1/attachments", tags=["attachments"]) + +MAX_SIZE = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +CHUNK_SIZE = 64 * 1024 + + +def build_attachment_response(att: Attachment) -> dict: + now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + return { + "id": str(att.id), + "chat_id": str(att.chat_id), + "owner_id": str(att.user_id), + "filename": att.filename, + "content_type": att.mime_type, + "size_bytes": att.size_bytes, + "sha256": att.sha256, + "status": "ready", + "variants": [], + "error_code": None, + "created_at": now, + "updated_at": now, + } + + +@router_upload.post("", status_code=201) +async def upload_attachment( + chat_id: uuid.UUID, + filename: str = Query(..., min_length=1, max_length=255, pattern=r"^[^/\\\x00]+$"), + request: Request = None, + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session), +): + result = await session.execute( + select(ChatMember).where( + ChatMember.chat_id == chat_id, + ChatMember.user_id == user.id, + ) + ) + if not result.scalar_one_or_none(): + raise HTTPException(404, "Чат не найден") + + sha_hasher = hashlib.sha256() + buffer = io.BytesIO() + total = 0 + first_bytes = b"" + + async for chunk in request.stream(): + if not chunk: + continue + total += len(chunk) + if total > MAX_SIZE: + raise HTTPException(413, "File too large") + + sha_hasher.update(chunk) + buffer.write(chunk) + + if len(first_bytes) < 262: + need = 262 - len(first_bytes) + first_bytes += chunk[:need] + + kind = filetype.guess(first_bytes) + if not kind or kind.mime not in ("image/png", "image/jpeg"): + raise HTTPException(400, "Only PNG/JPEG allowed") + + contents = buffer.getvalue() + try: + img = Image.open(io.BytesIO(contents)) + if img.width * img.height > MAX_PIXELS: + raise HTTPException(400, "Image too large (pixels)") + except HTTPException: + raise + except Exception: + raise HTTPException(400, "Corrupted image") + + object_key = f"{uuid.uuid4()}.{kind.extension}" + + minio_client.put_object( + BUCKET, + object_key, + io.BytesIO(contents), + len(contents), + content_type=kind.mime, + ) + + sha_hash = sha_hasher.hexdigest() + att = Attachment( + object_key=object_key, + filename=filename, + mime_type=kind.mime, + size_bytes=len(contents), + chat_id=chat_id, + user_id=user.id, + sha256=sha_hash, + ) + session.add(att) + await session.commit() + await session.refresh(att) + + return build_attachment_response(att) + + +@router_download.get("/{attachment_id}/content") +async def download_attachment_content( + attachment_id: uuid.UUID, + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session), +): + att = await session.get(Attachment, attachment_id) + if not att: + raise HTTPException(404, "Not found") + + result = await session.execute( + select(ChatMember).where( + ChatMember.chat_id == att.chat_id, + ChatMember.user_id == user.id, + ) + ) + if not result.scalar_one_or_none(): + raise HTTPException(404, "Not found") + + try: + response = minio_client.get_object(BUCKET, att.object_key) + return StreamingResponse( + response, + media_type=att.mime_type, + headers={"Content-Disposition": f'attachment; filename="{att.filename}"'}, + ) + except Exception: + raise HTTPException(404, "File not found in storage") + + +@router_download.get("/{attachment_id}") +async def get_attachment_metadata( + attachment_id: uuid.UUID, + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session), +): + att = await session.get(Attachment, attachment_id) + if not att: + raise HTTPException(404, "Not found") + + result = await session.execute( + select(ChatMember).where( + ChatMember.chat_id == att.chat_id, + ChatMember.user_id == user.id, + ) + ) + if not result.scalar_one_or_none(): + raise HTTPException(404, "Not found") + + return build_attachment_response(att) \ No newline at end of file diff --git a/app/routers/auth.py b/app/routers/auth.py new file mode 100644 index 00000000..7f2dfbcc --- /dev/null +++ b/app/routers/auth.py @@ -0,0 +1,84 @@ +from fastapi import APIRouter, Depends, HTTPException, status +from fastapi.security import HTTPBasic, HTTPBasicCredentials, HTTPAuthorizationCredentials +from pydantic import BaseModel, ConfigDict, field_serializer +from app.redis_client import redis_client +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +import uuid +from datetime import datetime + +from app.database import async_session_maker +from app.models.user import User +from app.security import verify_password, create_session, delete_session, bearer_scheme + +router = APIRouter(prefix="/api/v1/auth", tags=["auth"]) +security = HTTPBasic(realm="messenger") + + +class UserResponse(BaseModel): + id: uuid.UUID + username: str + display_name: str + created_at: datetime + model_config = ConfigDict(from_attributes=True) + + @field_serializer('created_at') + def serialize_dt(self, dt: datetime): + return dt.isoformat().replace("+00:00", "Z") + + +class SessionResponse(BaseModel): + token: str + token_type: str + expires_at: str + user: UserResponse + + @field_serializer('expires_at') + def serialize_dt(self, dt_str: str): + if dt_str and dt_str.endswith("+00:00"): + return dt_str.replace("+00:00", "Z") + return dt_str + + +# Basic Auth в Bearer Token +@router.post("/sessions", status_code=201, response_model=SessionResponse) +async def login(credentials: HTTPBasicCredentials = Depends(security)): + async with async_session_maker() as session: + result = await session.execute(select(User).where(User.username == credentials.username)) + user = result.scalar_one_or_none() + + if not user or not verify_password(credentials.password, user.password_hash): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect username or password", + headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}, + ) + + session_data = await create_session(user.id) + + session_data["user"] = user + + return session_data + +@router.delete("/sessions/current", status_code=204) +async def logout(credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme)): + if not credentials: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Not authenticated", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + token = credentials.credentials + + user_id = await redis_client.get(f"session:{token}") + if not user_id: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired token", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + await delete_session(token) + return None + diff --git a/app/routers/chats.py b/app/routers/chats.py new file mode 100644 index 00000000..64299117 --- /dev/null +++ b/app/routers/chats.py @@ -0,0 +1,139 @@ +import uuid +from datetime import datetime +from fastapi import APIRouter, HTTPException, Depends +from pydantic import BaseModel, Field, field_validator, ConfigDict, field_serializer +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select + +from app.models.chat import Chat, ChatMember +from app.models.user import User +from app.security import get_current_user +from app.dependencies import get_session +from app.pagination import paginate + +router = APIRouter(prefix="/api/v1/chats", tags=["chats"]) + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("member_ids") + @classmethod + def unique_members(cls, v): + if len(set(v)) != len(v): + raise ValueError("member_ids должны быть уникальными") + return v + +class ChatResponse(BaseModel): + id: uuid.UUID + title: str + member_ids: list[uuid.UUID] + created_at: datetime + model_config = ConfigDict(from_attributes=True) + + @field_serializer('created_at') + def serialize_dt(self, dt: datetime): + return dt.isoformat().replace("+00:00", "Z") + + +@router.post("", status_code=201) +async def create_chat( + payload: CreateChat, + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session) +): + if str(user.id) not in payload.member_ids: + raise HTTPException(status_code=400, detail="Пользователь должен быть участником чата") + + for member_id in payload.member_ids: + result = await session.execute(select(User).where(User.id == member_id)) + if not result.scalar_one_or_none(): + raise HTTPException(status_code=400, detail=f"несуществующий пользователь: {member_id}") + + chat = Chat(title=payload.title) + session.add(chat) + await session.flush() + + for member_id in payload.member_ids: + session.add(ChatMember(chat_id=chat.id, user_id=member_id)) + + await session.commit() + await session.refresh(chat) + + return { + "id": chat.id, + "title": chat.title, + "member_ids": payload.member_ids, + "created_at": chat.created_at.isoformat().replace("+00:00", "Z"), + } + + +@router.get("/{chat_id}") +async def get_chat( + chat_id: str, + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session) +): + result = await session.execute(select(Chat).where(Chat.id == chat_id)) + chat = result.scalar_one_or_none() + + if not chat: + raise HTTPException(status_code=404, detail="чат не найден") + + result = await session.execute( + select(ChatMember).where(ChatMember.chat_id == chat.id, ChatMember.user_id == user.id) + ) + if not result.scalar_one_or_none(): + raise HTTPException(status_code=404, detail="чат не найден") + + result = await session.execute(select(ChatMember.user_id).where(ChatMember.chat_id == chat.id)) + member_ids = [] + for row in result.all(): + user_id = row[0] + member_ids.append(str(user_id)) + + return { + "id": chat.id, + "title": chat.title, + "member_ids": member_ids, + "created_at": chat.created_at.isoformat().replace("+00:00", "Z"), + } + + +@router.get("") +async def list_chats( + limit: int = 20, + cursor: str | None = None, + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session) +): + if limit < 1 or limit > 100: + raise HTTPException(status_code=400, detail="limit должен быть от 1 до 100") + + result = await session.execute( + select(Chat) + .join(ChatMember, Chat.id == ChatMember.chat_id) + .where(ChatMember.user_id == user.id) + .order_by(Chat.created_at, Chat.id) + ) + chats = result.scalars().all() + + my_chats = [] + for chat in chats: + result = await session.execute(select(ChatMember.user_id).where(ChatMember.chat_id == chat.id)) + member_ids = [] + for row in result.all(): + user_id = row[0] + member_ids.append(str(user_id)) + + my_chats.append({ + "id": chat.id, + "title": chat.title, + "member_ids": member_ids, + "created_at": chat.created_at.isoformat().replace("+00:00", "Z"), + }) + + page, next_cursor = paginate(my_chats, limit, cursor) + return {"items": page, "next_cursor": next_cursor} \ No newline at end of file diff --git a/app/routers/health.py b/app/routers/health.py new file mode 100644 index 00000000..abcef176 --- /dev/null +++ b/app/routers/health.py @@ -0,0 +1,34 @@ +from fastapi import APIRouter +from fastapi import HTTPException +from sqlalchemy import text +from app.database import async_session_maker +from app.redis_client import redis_client + + +router = APIRouter(tags=["health"]) + + +@router.get("/health/live") +def live(): + return {"status": "ok"} + + +@router.get("/health/ready") +async def ready(): + try: + async with async_session_maker() as session: + await session.execute(text("SELECT 1")) + + await redis_client.ping() + + return {"status": "ok"} + except Exception: + raise HTTPException(status_code=503, detail="unavailable") + +@router.get("/") +def home(): + return { + "service": "MTUSI Messenger", + "version": "1.1.0", + "docs": "/docs", + } \ No newline at end of file diff --git a/app/routers/messages.py b/app/routers/messages.py new file mode 100644 index 00000000..baa2c11a --- /dev/null +++ b/app/routers/messages.py @@ -0,0 +1,181 @@ +import uuid +from datetime import datetime +from fastapi import APIRouter, HTTPException, Depends, Header +from pydantic import BaseModel, Field, ConfigDict, field_serializer, model_validator +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select +from app.redis_client import redis_client +import json + +from app.models.message import Message +from app.models.chat import Chat, ChatMember +from app.models.user import User +from app.models.attachment import Attachment +from app.security import get_current_user +from app.dependencies import get_session + +router = APIRouter(prefix="/api/v1/chats/{chat_id}/messages", tags=["messages"]) + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + text: str = Field(min_length=0, max_length=4000) + attachment_ids: list[uuid.UUID] = [] + + @model_validator(mode="after") + def check_text_or_attachment(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Сообщение должно содержать текст или вложение") + return self + + +class MessageResponse(BaseModel): + id: uuid.UUID + chat_id: uuid.UUID + sender_id: uuid.UUID + text: str + created_at: datetime + model_config = ConfigDict(from_attributes=True) + attachment_ids: list[uuid.UUID] = [] + + @field_serializer('created_at') + def serialize_dt(self, dt: datetime): + return dt.isoformat().replace("+00:00", "Z") + + +async def check_chat_access(chat_id: str, user_id: uuid.UUID, session: AsyncSession): + chat = await session.get(Chat, chat_id) + if not chat: + raise HTTPException(status_code=404, detail="чат не найден") + + result = await session.execute( + select(ChatMember).where(ChatMember.chat_id == chat.id, ChatMember.user_id == user_id) + ) + if not result.scalar_one_or_none(): + raise HTTPException(status_code=404, detail="чат не найден") + return chat + + +@router.post("", status_code=201, response_model=MessageResponse) +async def create_message( + chat_id: str, + payload: CreateMessage, + idempotency_key: str = Header( + ..., + alias="Idempotency-Key", + min_length=1, + max_length=128, + pattern=r"^[A-Za-z0-9._:-]+$", + ), + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session) +): + await check_chat_access(chat_id, user.id, session) + + if payload.attachment_ids: + result = await session.execute( + select(Attachment.id).where( + Attachment.id.in_(payload.attachment_ids), + Attachment.chat_id == chat_id, + ) + ) + found_ids = {str(row[0]) for row in result.all()} + requested_ids = {str(a) for a in payload.attachment_ids} + if found_ids != requested_ids: + raise HTTPException( + status_code=400, + detail="attachment_ids содержат файлы из другого чата или несуществующие" + ) + + redis_key = f"idem:{user.id}:{chat_id}:{idempotency_key}" + body = payload.model_dump(mode="json") + + raw = await redis_client.get(redis_key) + if raw: + stored = json.loads(raw) + if stored["state"] == "done": + if stored["request_body"] == body: + return stored["response"] + raise HTTPException(status_code=409, detail="Idempotency-Key переиспользован с другим телом запроса") + raise HTTPException(status_code=409, detail="Запрос с этим Idempotency-Key ещё обрабатывается") + + acquired = await redis_client.set( + redis_key, + json.dumps({"state": "processing", "request_body": body}), + nx=True, + ex=86400, + ) + if not acquired: + raise HTTPException(409, "Одновременный запрос с тем же самым Idempotency-Key") + + message = Message( + chat_id=chat_id, + sender_id=user.id, + text=payload.text, + attachment_ids=[str(a) for a in payload.attachment_ids] + ) + session.add(message) + await session.commit() + await session.refresh(message) + + response_data = { + "id": str(message.id), + "chat_id": str(message.chat_id), + "sender_id": str(message.sender_id), + "text": message.text, + "created_at": message.created_at.isoformat().replace("+00:00", "Z"), + "attachment_ids": message.attachment_ids, + } + + await redis_client.set( + redis_key, + json.dumps({"state": "done", + "request_body": body, + "response": response_data}), + ex=86400, + ) + + return response_data + + +@router.get("") +async def list_messages( + chat_id: str, + limit: int = 50, + cursor: str | None = None, + user: User = Depends(get_current_user), + session: AsyncSession = Depends(get_session) +): + if limit < 1 or limit > 100: + raise HTTPException(status_code=400, detail="limit должен быть от 1 до 100") + + await check_chat_access(chat_id, user.id, session) + + offset = int(cursor) if cursor else 0 + + result = await session.execute( + select(Message) + .where(Message.chat_id == chat_id) + .order_by(Message.created_at, Message.id) + .limit(limit + 1) + .offset(offset) + ) + msgs = result.scalars().all() + + has_next = len(msgs) > limit + if has_next: + msgs = msgs[:limit] + + my_messages = [ + { + "id": m.id, + "chat_id": m.chat_id, + "sender_id": m.sender_id, + "text": m.text, + "created_at": m.created_at.isoformat().replace("+00:00", "Z"), + "attachment_ids": m.attachment_ids, + } for m in msgs + ] + + next_cursor = str(offset + limit) if has_next else None + return {"items": my_messages, "next_cursor": next_cursor} \ No newline at end of file diff --git a/app/routers/users.py b/app/routers/users.py new file mode 100644 index 00000000..cc77b513 --- /dev/null +++ b/app/routers/users.py @@ -0,0 +1,55 @@ +import uuid +from datetime import datetime +from fastapi import APIRouter, HTTPException, Depends +from pydantic import BaseModel, Field, ConfigDict, field_serializer +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from app.models.user import User +from app.security import get_password_hash, get_current_user +from app.dependencies import get_session + +router = APIRouter(prefix="/api/v1/users", tags=["users"]) + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=5, max_length=128) + +class UserResponse(BaseModel): + id: uuid.UUID + username: str + display_name: str + created_at: datetime + model_config = ConfigDict(from_attributes=True) + + @field_serializer('created_at') + def serialize_dt(self, dt: datetime): + return dt.isoformat().replace("+00:00", "Z") + +@router.post("", status_code=201, response_model=UserResponse) +async def register_user(payload: CreateUser, session: AsyncSession = Depends(get_session)): + result = await session.execute(select(User).where(User.username == payload.username)) + if result.scalar_one_or_none(): + raise HTTPException(status_code=409, detail="Имя пользователя уже существует") + + user = User( + username=payload.username, + display_name=payload.display_name, + password_hash=get_password_hash(payload.password) + ) + session.add(user) + + try: + await session.commit() + except IntegrityError: + await session.rollback() + raise HTTPException(status_code=409, detail="Имя пользователя уже существует") + + await session.refresh(user) + return user + +@router.get("/me", response_model=UserResponse) +async def current_user(user: User = Depends(get_current_user)): + return user \ No newline at end of file diff --git a/app/security.py b/app/security.py new file mode 100644 index 00000000..e6f5588f --- /dev/null +++ b/app/security.py @@ -0,0 +1,81 @@ +import secrets +from datetime import datetime, timezone, timedelta +from passlib.context import CryptContext +from fastapi import Depends, HTTPException, status +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from sqlalchemy import select +from app.database import async_session_maker +from app.models.user import User +from app.redis_client import redis_client +from app.config import SESSION_TTL_SECONDS + +pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") +bearer_scheme = HTTPBearer(auto_error=False) + +def get_password_hash(password: str) -> str: + return pwd_context.hash(password) + + +def verify_password(plain: str, hashed: str) -> bool: + return pwd_context.verify(plain, hashed) + + +async def create_session(user_id: str) -> dict: + token = secrets.token_urlsafe(32) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS) + + await redis_client.setex( + name=f"session:{token}", + time=SESSION_TTL_SECONDS, + value=str(user_id), + ) + + return { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at.isoformat().replace("+00:00", "Z"), + } + + +async def delete_session(token: str) -> None: + await redis_client.delete(f"session:{token}") + + +async def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme)): + if not credentials: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Not authenticated", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + token = credentials.credentials + + + try: + user_id = await redis_client.get(f"session:{token}") + except Exception: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Внутренняя ошибка сервера" + ) + + if not user_id: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired token", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + async with async_session_maker() as session: + result = await session.execute(select(User).where(User.id == user_id)) + user = result.scalar_one_or_none() + + if not user: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="User not found", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + return user \ No newline at end of file diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 00000000..5abf57a6 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,22 @@ +import threading +from minio import Minio +from app.config import S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_BUCKET + +users_lock = threading.Lock() +messages_lock = threading.Lock() + +users_by_id: dict[str, dict] = {} +users_by_username: dict[str, dict] = {} +chats_by_id: dict[str, dict] = {} +messages_by_chat: dict[str, list[dict]] = {} + +endpoint = S3_ENDPOINT.replace("http://", "").replace("https://", "") + +minio_client = Minio( + endpoint, + access_key=S3_ACCESS_KEY_ID, + secret_key=S3_SECRET_ACCESS_KEY, + secure=False, +) + +BUCKET = S3_BUCKET \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..ac874c0d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,142 @@ +services: + db: + image: postgres:16-alpine + container_name: mtusi_messenger_db + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + command: postgres -c synchronous_commit=off + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + + redis: + image: redis:7-alpine + container_name: mtusi_messenger_redis + command: redis-server --appendonly yes + volumes: + - redis_data:/data + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + restart: unless-stopped + + app1: + build: . + environment: + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS} + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + minio: + condition: service_healthy + volumes: + - ./app:/app/app + - ./alembic:/app/alembic + - ./alembic.ini:/app/alembic.ini + + app2: + build: . + environment: + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS} + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + minio: + condition: service_healthy + volumes: + - ./app:/app/app + - ./alembic:/app/alembic + - ./alembic.ini:/app/alembic.ini + + nginx: + image: nginx:alpine + container_name: mtusi_messenger_nginx + ports: + - "127.0.0.1:8080:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + - app1 + - app2 + + minio: + image: pgsty/silo:latest + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + volumes: + - minio_data:/data + ports: + - "127.0.0.1:9000:9000" + - "127.0.0.1:9001:9001" + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/ready" ] + interval: 5s + timeout: 5s + retries: 5 + + minio-init: + image: pgsty/mc:latest + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9000 minioadmin minioadmin; + /usr/bin/mc mb --ignore-existing myminio/${S3_BUCKET}; + /usr/bin/mc anonymous set none myminio/${S3_BUCKET}; + /usr/bin/mc admin user add myminio api-user api-user-secret-2026 || true; + /usr/bin/mc admin policy attach myminio readwrite --user api-user || true; + exit 0; + " + +volumes: + postgres_data: + redis_data: + minio_data: \ No newline at end of file diff --git a/evidence/explain_composite.txt b/evidence/explain_composite.txt new file mode 100644 index 00000000..f102243f --- /dev/null +++ b/evidence/explain_composite.txt @@ -0,0 +1,13 @@ + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=0.42..5.54 rows=50 width=66) (actual time=0.103..0.111 rows=50 loops=1) + Buffers: shared hit=4 + -> Index Scan using ix_messages_chat_created_id on messages (cost=0.42..8222.03 rows=80203 width=66) (actual time=0.102..0.107 rows=50 loops=1) + Index Cond: (chat_id = '66666666-6666-4666-8666-666666666666'::uuid) + Buffers: shared hit=4 + Planning: + Buffers: shared hit=107 + Planning Time: 0.865 ms + Execution Time: 0.168 ms +(9 rows) + diff --git a/evidence/explain_simple.txt b/evidence/explain_simple.txt new file mode 100644 index 00000000..b8257823 --- /dev/null +++ b/evidence/explain_simple.txt @@ -0,0 +1,22 @@ + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=6279.67..6285.42 rows=50 width=66) (actual time=7.323..8.152 rows=50 loops=1) + Buffers: shared hit=1063 read=72 + -> Gather Merge (cost=6279.67..11696.05 rows=47099 width=66) (actual time=7.322..8.148 rows=50 loops=1) + Workers Planned: 1 + Workers Launched: 1 + Buffers: shared hit=1063 read=72 + -> Sort (cost=5279.66..5397.40 rows=47099 width=66) (actual time=6.368..6.368 rows=26 loops=2) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 38kB + Buffers: shared hit=1063 read=72 + Worker 0: Sort Method: top-N heapsort Memory: 38kB + -> Parallel Index Scan using ix_messages_chat_only on messages (cost=0.42..3715.06 rows=47099 width=66) (actual time=0.031..2.837 rows=40000 loops=2) +--More-- Index Cond: (chat_id = '66666666-6666-4666-8666-66666666666--More-- 6'::uuid) +--More-- Buffers: shared hit=1048 read=72 +--More-- Planning: +--More-- Buffers: shared hit=98 read=1 +--More-- Planning Time: 0.335 ms +--More-- Execution Time: 8.200 ms +--More-- (18 rows) +--More-- diff --git a/evidence/failover-app1.txt b/evidence/failover-app1.txt new file mode 100644 index 00000000..514bb4e2 --- /dev/null +++ b/evidence/failover-app1.txt @@ -0,0 +1,29 @@ +./scripts/failover-test.sh app1 +Останавливаю app1 +16:50:48.3N +Останавливаю app1... +app1 остановлен + +Шлю запросы каждые 0.5 сек, смотрю код и x-instance-id +16:50:48.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:50:51.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:50:51.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:50:52.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:50:52.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:50:53.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:50:53.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:50:54.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:50:55.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:50:55.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:50:56.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:50:58.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:50:59.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:50:59.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:51:00.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:51:00.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:51:01.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:51:02.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:51:02.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b +16:51:03.3N code=200 x-instance-id: 4749a273-c09f-47b2-9172-f373b786233b + +Поднимаю app1 обратно diff --git a/evidence/failover-app2.txt b/evidence/failover-app2.txt new file mode 100644 index 00000000..98fd48ee --- /dev/null +++ b/evidence/failover-app2.txt @@ -0,0 +1,29 @@ +./scripts/failover-test.sh app2 +Останавливаю app2 +16:51:08.3N +Останавливаю app2... +app2 остановлен + +Шлю запросы каждые 0.5 сек, смотрю код и x-instance-id +16:51:09.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:11.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:14.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:14.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:15.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:15.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:16.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:17.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:17.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:18.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:18.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:19.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:21.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:22.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:22.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:23.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:24.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:24.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:25.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae +16:51:25.3N code=200 x-instance-id: 534c1846-b123-481b-a9c4-71c7049315ae + +Поднимаю app2 обратно diff --git a/evidence/filename-traversal.txt b/evidence/filename-traversal.txt new file mode 100644 index 00000000..34a6ef7e --- /dev/null +++ b/evidence/filename-traversal.txt @@ -0,0 +1,10 @@ +HTTP/1.1 400 Bad Request +Server: nginx/1.29.4 +Date: Wed, 11 Nov 2026 17:04:13 GMT +Content-Type: application/json +Content-Length: 145 +Connection: keep-alive +x-request-id: 69432ca6-ed29-4dbd-a76c-18a468b47182 +x-instance-id: 9080c907-3eed-49de-b870-092b89ee4eb4 + +{"error":{"code":"invalid_request","message":"Ошибка валидации данных","request_id":"69432ca6-ed29-4dbd-a76c-18a468b47182"}} \ No newline at end of file diff --git a/evidence/load-1replica.txt b/evidence/load-1replica.txt new file mode 100644 index 00000000..b2f1043d --- /dev/null +++ b/evidence/load-1replica.txt @@ -0,0 +1,18 @@ +python3 tests/load.py --base-url "http://localhost:8080" +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30, + "elapsed_seconds": 30.609, + "concurrency": 4, + "successful_requests": 8195, + "failed_requests": 8, + "error_ratio": 0.000975, + "successful_rps": 267.73, + "successful_latency_ms": { + "p50": 9.403, + "p95": 12.088, + "p99": 17.671 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-2replicas-2cpu-conc20-leastconn.txt b/evidence/load-2replicas-2cpu-conc20-leastconn.txt new file mode 100644 index 00000000..1c718f0b --- /dev/null +++ b/evidence/load-2replicas-2cpu-conc20-leastconn.txt @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30, + "elapsed_seconds": 32.212, + "concurrency": 20, + "successful_requests": 7940, + "failed_requests": 60, + "error_ratio": 0.0075, + "successful_rps": 246.491, + "successful_latency_ms": { + "p50": 32.546, + "p95": 76.906, + "p99": 114.623 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-2replicas.txt b/evidence/load-2replicas.txt new file mode 100644 index 00000000..1107c6b1 --- /dev/null +++ b/evidence/load-2replicas.txt @@ -0,0 +1,22 @@ +=== Замер для 2 реплик === +Лимиты: cpus=0.5, memory=512M на реплику (суммарно 1 CPU / 1 GB) +Конфигурация: app1 + app2 за Nginx +Нагрузка: make load (POST /messages, 30 сек, concurrency 4) + +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30, + "elapsed_seconds": 31.155, + "concurrency": 4, + "successful_requests": 8156, + "failed_requests": 3, + "error_ratio": 0.000368, + "successful_rps": 261.789, + "successful_latency_ms": { + "p50": 8.613, + "p95": 36.591, + "p99": 48.722 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-3replicas-2cpu-conc20-pool20.txt b/evidence/load-3replicas-2cpu-conc20-pool20.txt new file mode 100644 index 00000000..4a74d9d0 --- /dev/null +++ b/evidence/load-3replicas-2cpu-conc20-pool20.txt @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30, + "elapsed_seconds": 33.193, + "concurrency": 20, + "successful_requests": 8386, + "failed_requests": 60, + "error_ratio": 0.007104, + "successful_rps": 252.642, + "successful_latency_ms": { + "p50": 18.858, + "p95": 78.217, + "p99": 156.447 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-3replicas-2cpu-conc4.txt b/evidence/load-3replicas-2cpu-conc4.txt new file mode 100644 index 00000000..7dc270e3 --- /dev/null +++ b/evidence/load-3replicas-2cpu-conc4.txt @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30, + "elapsed_seconds": 31.089, + "concurrency": 4, + "successful_requests": 8190, + "failed_requests": 12, + "error_ratio": 0.001463, + "successful_rps": 263.441, + "successful_latency_ms": { + "p50": 7.843, + "p95": 11.567, + "p99": 16.163 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-4replicas-2cpu-conc20.txt b/evidence/load-4replicas-2cpu-conc20.txt new file mode 100644 index 00000000..0fbeb33a --- /dev/null +++ b/evidence/load-4replicas-2cpu-conc20.txt @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30, + "elapsed_seconds": 32.952, + "concurrency": 20, + "successful_requests": 8299, + "failed_requests": 20, + "error_ratio": 0.002404, + "successful_rps": 251.855, + "successful_latency_ms": { + "p50": 9.001, + "p95": 274.971, + "p99": 389.683 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-4replicas-2cpu.txt b/evidence/load-4replicas-2cpu.txt new file mode 100644 index 00000000..6ad5bb2a --- /dev/null +++ b/evidence/load-4replicas-2cpu.txt @@ -0,0 +1,18 @@ +python3 tests/load.py --base-url "http://localhost:8080" +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30, + "elapsed_seconds": 30.999, + "concurrency": 4, + "successful_requests": 8165, + "failed_requests": 12, + "error_ratio": 0.001468, + "successful_rps": 263.396, + "successful_latency_ms": { + "p50": 7.054, + "p95": 11.933, + "p99": 23.37 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-4replicas.txt b/evidence/load-4replicas.txt new file mode 100644 index 00000000..fe97b10c --- /dev/null +++ b/evidence/load-4replicas.txt @@ -0,0 +1,18 @@ +python3 tests/load.py --base-url "http://localhost:8080" +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30, + "elapsed_seconds": 32.012, + "concurrency": 4, + "successful_requests": 8160, + "failed_requests": 2, + "error_ratio": 0.000245, + "successful_rps": 254.903, + "successful_latency_ms": { + "p50": 6.556, + "p95": 53.415, + "p99": 69.04 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/private-bucket.txt b/evidence/private-bucket.txt new file mode 100644 index 00000000..4f0733d8 --- /dev/null +++ b/evidence/private-bucket.txt @@ -0,0 +1,25 @@ +=== Проверка private bucket === +Object key: de86cd68-f019-4b7b-83f7-47d9d2fcd03a.png + +=== Прямой GET без авторизации (ожидаем 403) === + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 413 100 413 0 0 132k 0 --:--:-- --:--:-- --:--:-- 134k +HTTP/1.1 403 Forbidden +Accept-Ranges: bytes +Content-Length: 413 +Content-Type: application/xml +Server: Silo +Strict-Transport-Security: max-age=31536000; includeSubDomains +Vary: Origin +Vary: Accept-Encoding +X-Amz-Id-2: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8 +X-Amz-Request-Id: 18E75A22BC9CBD47 +X-Content-Type-Options: nosniff +X-Ratelimit-Limit: 2160 +X-Ratelimit-Remaining: 2160 +X-Xss-Protection: 1; mode=block +Date: Thu, 12 Nov 2026 16:48:01 GMT + + +AccessDeniedAccess Denied.de86cd68-f019-4b7b-83f7-47d9d2fcd03a.pngmessenger-attachments/messenger-attachments/de86cd68-f019-4b7b-83f7-47d9d2fcd03a.png18E75A22BC9CBD47dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8 \ No newline at end of file diff --git a/evidence/ram-experiment.txt b/evidence/ram-experiment.txt new file mode 100644 index 00000000..f0e3cd42 --- /dev/null +++ b/evidence/ram-experiment.txt @@ -0,0 +1,28 @@ +NAME MEM USAGE / LIMIT CPU % +mtusi2026lab4-app2-1 94.89MiB / 512MiB 0.21% +mtusi2026lab4-app1-1 95.96MiB / 512MiB 0.18% +mtusi_messenger_nginx 2.594MiB / 8.722GiB 0.00% +mtusi_messenger_db 46.91MiB / 8.722GiB 0.75% +mtusi2026lab4-minio-1 271.8MiB / 8.722GiB 1.31% +mtusi_messenger_redis 9.613MiB / 8.722GiB 0.87% +RAM Experiment: 10 concurrent uploads, ~5 MB each +Chat ID: 55d6b9f8-5fb5-4771-b4a6-47dee2886c12 +Генерируем 10 картинок (шум, ~5 МБ)... +Размеры: min=5.62, max=7.15, avg=6.37 МБ +Суммарно: 63.72 МБ + +Запускаем 10 одновременных загрузок... + +=== Результаты === +Всего: 10 +Успешных (201): 10 +Ошибок: 0 +Latency: min=0.65s, max=0.88s, avg=0.81s +Общее время: 0.89s +NAME MEM USAGE / LIMIT CPU % +mtusi2026lab4-app2-1 94.9MiB / 512MiB 4.69% +mtusi2026lab4-app1-1 161.7MiB / 512MiB 8.94% +mtusi_messenger_nginx 2.594MiB / 8.722GiB 0.00% +mtusi_messenger_db 54.61MiB / 8.722GiB 0.03% +mtusi2026lab4-minio-1 280.1MiB / 8.722GiB 0.06% +mtusi_messenger_redis 9.617MiB / 8.722GiB 0.75% diff --git a/evidence/reconciler-full.txt b/evidence/reconciler-full.txt new file mode 100644 index 00000000..802610f7 --- /dev/null +++ b/evidence/reconciler-full.txt @@ -0,0 +1,122 @@ +================================================================ + RECONCILER TEST — Thu Nov 12 19:44:47 MSK 2026 +================================================================ + +=== 1. Создание пользователя (reconcile_tester_1794501887) === +HTTP 201 +Body: {"id":"eeb6a83d-a4ad-4fba-a2c3-38eefe0e0e1d","username":"reconcile_tester_1794501887","display_name":"Reconciler Test","created_at":"2026-11-12T16:44:47.844572Z"} + +=== 2. Логин === +HTTP 201 +Токен получен: xluXtVfFzEGG7Mvi0C9g... + +User ID: eeb6a83d-a4ad-4fba-a2c3-38eefe0e0e1d + +=== 3. Создание чата === +Chat ID: 52d0da4c-7af8-4548-8dcf-b3be6badcae6 + +=== 4. Создаём ORPHAN === +Orphan attachment_id: 0bbfa860-f5f5-42a5-9611-29f55f5baafd +Orphan object_key: aec0da17-d643-41ce-a3f0-d8a83e96ad97.png +Запись удалена из БД — ORPHAN + +=== 5. Создаём MISSING === +Missing attachment_id: 414ecbba-4119-414f-8796-8c41128f8e70 +Missing object_key: 43dbaa2d-2f62-44cd-9d27-2aec1bb36164.png +Removed `myminio/messenger-attachments/43dbaa2d-2f62-44cd-9d27-2aec1bb36164.png`. +Файл удалён из MinIO — MISSING + +=== Пауза 11 секунд (orphan станет старше grace_period 10 сек) === + +=== 6. DRY-RUN (ожидаем ORPHAN + MISSING) === +/app/scripts/reconcile.py:81: DeprecationWarning: Parsing dates involving a day of month without a year specified is ambiguious +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647. + minio_client.stat_object(BUCKET, key) +Поиск orphan объектов в S3, файлы без записи в БД +Файлов в БД: 102 +Объектов в MinIO: 99 + [ORPHAN] aec0da17-d643-41ce-a3f0-d8a83e96ad97.png (возраст: 0:00:12.293643, старше grace_period) +[DRY-RUN] Найдено 1 orphan. Удаление не выполнено. + +Поиск missing объектов, записи в БД без файла в MinIO +Записей в БД: 102 + [SKIP] id=ec42f559-a4ed-4237-87bd-8ed4791cbb12, key=33655bad-fe2d-4144-b621-b667d8eee105.png — используется в сообщении, не удаляем + [SKIP] id=57462036-5532-4a23-8a84-4437c1e89c64, key=aa776ea9-b469-4e4c-b5af-6f0e2b514a12.png — используется в сообщении, не удаляем + [SKIP] id=0858aa8f-d096-4ac3-ae8f-68b66411fe2e, key=f120b3d7-f213-42ea-ab3e-9b837a0167a0.png — используется в сообщении, не удаляем + [MISSING] id=414ecbba-4119-414f-8796-8c41128f8e70, key=43dbaa2d-2f62-44cd-9d27-2aec1bb36164.png — не используется, можно удалить +[DRY-RUN] Missing: 1 (удалить), 3 (используется, SKIP). Удаление не выполнено. + +Итог +Orphan S3 без БД: 1 +Missing БД без S3 (удалено): 1 +Режим: DRY-RUN (ничего не удалено) + +=== 7. EXECUTE === +/app/scripts/reconcile.py:81: DeprecationWarning: Parsing dates involving a day of month without a year specified is ambiguious +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647. + minio_client.stat_object(BUCKET, key) +Поиск orphan объектов в S3, файлы без записи в БД +Файлов в БД: 102 +Объектов в MinIO: 99 + [ORPHAN] aec0da17-d643-41ce-a3f0-d8a83e96ad97.png (возраст: 0:00:13.113091, старше grace_period) + [DELETED] aec0da17-d643-41ce-a3f0-d8a83e96ad97.png +Удалено 1 orphan-объектов. + +Поиск missing объектов, записи в БД без файла в MinIO +Записей в БД: 102 + [SKIP] id=ec42f559-a4ed-4237-87bd-8ed4791cbb12, key=33655bad-fe2d-4144-b621-b667d8eee105.png — используется в сообщении, не удаляем + [SKIP] id=57462036-5532-4a23-8a84-4437c1e89c64, key=aa776ea9-b469-4e4c-b5af-6f0e2b514a12.png — используется в сообщении, не удаляем + [SKIP] id=0858aa8f-d096-4ac3-ae8f-68b66411fe2e, key=f120b3d7-f213-42ea-ab3e-9b837a0167a0.png — используется в сообщении, не удаляем + [MISSING] id=414ecbba-4119-414f-8796-8c41128f8e70, key=43dbaa2d-2f62-44cd-9d27-2aec1bb36164.png — не используется, можно удалить +Удалено 1 неиспользуемых missing из БД. + [SKIP] 3 используемых missing — не удалены (защита от битых сообщений). + +Итог +Orphan S3 без БД: 1 +Missing БД без S3 (удалено): 1 +Режим: EXECUTE (изменения применены) + +=== 8. Проверка: orphan удалён из MinIO === +Orphan в MinIO: mc: Unable to stat `myminio/messenger-attachments/aec0da17-d643-41ce-a3f0-d8a83e96ad97.png`. Object does not exist. + +=== 9. Проверка: missing удалён из БД === +Записей с id=414ecbba-4119-414f-8796-8c41128f8e70 в БД: 0 + +=== 10. Идемпотентность: повторный DRY-RUN === +/app/scripts/reconcile.py:81: DeprecationWarning: Parsing dates involving a day of month without a year specified is ambiguious +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647. + minio_client.stat_object(BUCKET, key) +Поиск orphan объектов в S3, файлы без записи в БД +Файлов в БД: 101 +Объектов в MinIO: 98 +Orphan-объектов не найдено. + +Поиск missing объектов, записи в БД без файла в MinIO +Записей в БД: 101 + [SKIP] id=ec42f559-a4ed-4237-87bd-8ed4791cbb12, key=33655bad-fe2d-4144-b621-b667d8eee105.png — используется в сообщении, не удаляем + [SKIP] id=57462036-5532-4a23-8a84-4437c1e89c64, key=aa776ea9-b469-4e4c-b5af-6f0e2b514a12.png — используется в сообщении, не удаляем + [SKIP] id=0858aa8f-d096-4ac3-ae8f-68b66411fe2e, key=f120b3d7-f213-42ea-ab3e-9b837a0167a0.png — используется в сообщении, не удаляем +[DRY-RUN] Missing: 0 (удалить), 3 (используется, SKIP). Удаление не выполнено. + +Итог +Orphan S3 без БД: 0 +Missing БД без S3 (удалено): 0 +Режим: DRY-RUN (ничего не удалено) + +================================================================ + ИТОГ +================================================================ +PASS: + - Пользователь reconcile_tester_1794501887 создан (HTTP 201), логин 201, токен получен + - ORPHAN удалён из MinIO + - MISSING удалён из БД + - Повторный dry-run: 0/0 (идемпотентность) diff --git a/evidence/reconciler.txt b/evidence/reconciler.txt new file mode 100644 index 00000000..e69de29b diff --git a/evidence/rolling-load.txt b/evidence/rolling-load.txt new file mode 100644 index 00000000..3ba1bfdd --- /dev/null +++ b/evidence/rolling-load.txt @@ -0,0 +1,32 @@ +./scripts/rolling-load.sh +Нагрузка: 30 запросов с интервалом 0.5 сек +16:51:30 code=200 instance=534c1846-b123-481b-a9c4-71c7049315ae +16:51:30 code=200 instance=0a6121d9-07cc-4491-9937-e8b68e5156c5 +16:51:31 code=200 instance=0a6121d9-07cc-4491-9937-e8b68e5156c5 +16:51:33 code=200 instance=0a6121d9-07cc-4491-9937-e8b68e5156c5 +16:51:34 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:35 code=200 instance=0a6121d9-07cc-4491-9937-e8b68e5156c5 +16:51:35 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:36 code=200 instance=0a6121d9-07cc-4491-9937-e8b68e5156c5 +16:51:36 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:37 code=200 instance=0a6121d9-07cc-4491-9937-e8b68e5156c5 +16:51:37 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:38 code=200 instance=0a6121d9-07cc-4491-9937-e8b68e5156c5 +16:51:38 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:39 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:40 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:41 code=200 instance=86e75a44-8448-4aec-99c1-a2d9378f0dc2 +16:51:41 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:42 code=200 instance=86e75a44-8448-4aec-99c1-a2d9378f0dc2 +16:51:43 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:43 code=200 instance=86e75a44-8448-4aec-99c1-a2d9378f0dc2 +16:51:44 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:44 code=200 instance=86e75a44-8448-4aec-99c1-a2d9378f0dc2 +16:51:45 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:45 code=200 instance=86e75a44-8448-4aec-99c1-a2d9378f0dc2 +16:51:46 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:46 code=200 instance=86e75a44-8448-4aec-99c1-a2d9378f0dc2 +16:51:47 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:47 code=200 instance=86e75a44-8448-4aec-99c1-a2d9378f0dc2 +16:51:48 code=200 instance=06026485-762d-4e6a-a368-e1bac311a756 +16:51:49 code=200 instance=86e75a44-8448-4aec-99c1-a2d9378f0dc2 diff --git a/evidence/rolling-update.txt b/evidence/rolling-update.txt new file mode 100644 index 00000000..59e3c9e7 --- /dev/null +++ b/evidence/rolling-update.txt @@ -0,0 +1,159 @@ +./scripts/rolling-update.sh +Rolling update: app1 + Container mtusi2026lab3-app1-1 Stopping + Container mtusi2026lab3-app1-1 Stopped +#0 building with "desktop-linux" instance using docker driver + +#1 [app1 internal] load build definition from Dockerfile +#1 transferring dockerfile: 685B done +#1 DONE 0.0s + +#2 [app1 internal] load metadata for docker.io/library/python:3.13-slim +#2 DONE 1.8s + +#3 [app1 internal] load .dockerignore +#3 transferring context: 279B done +#3 DONE 0.0s + +#4 [app1 1/9] FROM docker.io/library/python:3.13-slim@sha256:09a17cad02668ae61ece136b4fd36ac3849e14f670ebedb637ea3e15580525bc +#4 DONE 0.0s + +#5 [app1 internal] load build context +#5 transferring context: 4.73kB done +#5 DONE 0.0s + +#6 [app1 4/9] RUN pip install --no-cache-dir -r requirements-dev.txt +#6 CACHED + +#7 [app1 3/9] COPY requirements-dev.txt . +#7 CACHED + +#8 [app1 2/9] WORKDIR /app +#8 CACHED + +#9 [app1 8/9] COPY alembic/ alembic/ +#9 CACHED + +#10 [app1 6/9] COPY app/ app/ +#10 CACHED + +#11 [app1 7/9] COPY alembic.ini . +#11 CACHED + +#12 [app1 5/9] RUN useradd --create-home --shell /bin/bash appuser +#12 CACHED + +#13 [app1 9/9] RUN chown -R appuser:appuser /app/alembic +#13 CACHED + +#14 [app1] exporting to image +#14 exporting layers done +#14 writing image sha256:af1621af7e8af7324cc026feef618c950fa872eeb17c4333dade10b720a57503 done +#14 naming to docker.io/library/mtusi2026lab3-app1 done +#14 DONE 0.0s + +#15 [app1] resolving provenance for metadata file +#15 DONE 0.0s + Container mtusi_messenger_redis Running + Container mtusi_messenger_db Running + Container mtusi2026lab3-app1-1 Created + Container mtusi_messenger_redis Waiting + Container mtusi_messenger_db Waiting + Container mtusi_messenger_redis Healthy + Container mtusi_messenger_db Healthy + Container mtusi2026lab3-app1-1 Starting + Container mtusi2026lab3-app1-1 Started +Жду, пока app1 станет healthy... + +Rolling update: app2 + Container mtusi2026lab3-app2-1 Stopping + Container mtusi2026lab3-app2-1 Stopped +#0 building with "desktop-linux" instance using docker driver + +#1 [app2 internal] load build definition from Dockerfile +#1 transferring dockerfile: 685B done +#1 DONE 0.0s + +#2 [app2 internal] load metadata for docker.io/library/python:3.13-slim +#2 DONE 0.3s + +#3 [app2 internal] load .dockerignore +#3 transferring context: 279B done +#3 DONE 0.0s + +#4 [app2 1/9] FROM docker.io/library/python:3.13-slim@sha256:09a17cad02668ae61ece136b4fd36ac3849e14f670ebedb637ea3e15580525bc +#4 DONE 0.0s + +#5 [app2 internal] load build context +#5 transferring context: 4.73kB done +#5 DONE 0.0s + +#6 [app2 3/9] COPY requirements-dev.txt . +#6 CACHED + +#7 [app2 7/9] COPY alembic.ini . +#7 CACHED + +#8 [app2 8/9] COPY alembic/ alembic/ +#8 CACHED + +#9 [app2 5/9] RUN useradd --create-home --shell /bin/bash appuser +#9 CACHED + +#10 [app2 2/9] WORKDIR /app +#10 CACHED + +#11 [app2 4/9] RUN pip install --no-cache-dir -r requirements-dev.txt +#11 CACHED + +#12 [app2 6/9] COPY app/ app/ +#12 CACHED + +#13 [app2 9/9] RUN chown -R appuser:appuser /app/alembic +#13 CACHED + +#14 [app2] exporting to image +#14 exporting layers done +#14 writing image sha256:9c5ffbef790bf603d2cd9d54e18df8b05aa9541f5201974646cfd5fb1d46720b done +#14 naming to docker.io/library/mtusi2026lab3-app2 done +#14 DONE 0.0s + +#15 [app2] resolving provenance for metadata file +#15 DONE 0.0s + Container mtusi_messenger_redis Running + Container mtusi_messenger_db Running + Container mtusi2026lab3-app2-1 Created + Container mtusi_messenger_redis Waiting + Container mtusi_messenger_db Waiting + Container mtusi_messenger_db Healthy + Container mtusi_messenger_redis Healthy + Container mtusi2026lab3-app2-1 Starting + Container mtusi2026lab3-app2-1 Started +Жду, пока app2 станет healthy... + +Проверка: обе реплики живы + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 15 100 15 0 0 2824 0 --:--:-- --:--:-- --:--:-- 3000 +HTTP/1.1 200 OK +Server: nginx/1.29.4 +Date: Fri, 06 Nov 2026 13:51:44 GMT +Content-Type: application/json +Content-Length: 15 +Connection: keep-alive +x-instance-id: 06026485-762d-4e6a-a368-e1bac311a756 +x-request-id: 9fd1f3e6-0c5a-4cce-a727-35467ef553b8 + +{"status":"ok"} % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 15 100 15 0 0 3764 0 --:--:-- --:--:-- --:--:-- 5000 +HTTP/1.1 200 OK +Server: nginx/1.29.4 +Date: Fri, 06 Nov 2026 13:51:44 GMT +Content-Type: application/json +Content-Length: 15 +Connection: keep-alive +x-instance-id: 86e75a44-8448-4aec-99c1-a2d9378f0dc2 +x-request-id: 23fcdcf5-8f9c-48f3-8973-4c5a49186e23 + +{"status":"ok"} \ No newline at end of file diff --git a/evidence/run-reconciler-test.sh b/evidence/run-reconciler-test.sh new file mode 100644 index 00000000..37c071f7 --- /dev/null +++ b/evidence/run-reconciler-test.sh @@ -0,0 +1,99 @@ +set -e +mkdir -p evidence + +{ +echo "================================================================" +echo " RECONCILER TEST — $(date)" +echo "================================================================" +echo "" + +echo "=== 1. Логин ===" +curl -s -X POST http://localhost:8080/api/v1/users \ + -H "Content-Type: application/json" \ + -d '{"username": "reconcile_tester", "display_name": "Reconciler Test", "password": "secret1234567"}' > /dev/null + +TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/sessions \ + -u "reconcile_tester:secret1234567" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['token'])") +echo "Токен получен: ${TOKEN:0:20}..." + +USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \ + -H "Authorization: Bearer $TOKEN" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +echo "User ID: $USER_ID" +echo "" + +echo "=== 2. Создание чата ===" +CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"title\": \"Reconcile Test\", \"member_ids\": [\"$USER_ID\"]}" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +echo "Chat ID: $CHAT_ID" +echo "" + +echo "=== 3. Загрузка файла (файл → MinIO, метаданные → БД) ===" +RESP=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=reconcile-test.png" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: image/png" \ + --data-binary @tests/fixtures/sample.png) + +echo "Ответ на загрузку: $RESP" +ATT_ID=$(echo "$RESP" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +echo "Attachment ID: $ATT_ID" +echo "" + +echo "=== 4. Получаем object_key из БД ===" +OBJECT_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT object_key FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n') +echo "Object key: $OBJECT_KEY" +echo "" + +echo "=== 5. Удаляем файл из MinIO (создаём MISSING) ===" +docker compose exec -T minio sh -c \ + "mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc rm myminio/messenger-attachments/$OBJECT_KEY" +echo "Файл удалён из MinIO. В БД запись осталась — это MISSING." +echo "" + +echo "=== 6. RECONCILER — DRY-RUN (должен найти MISSING, не удалять) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 \ + python scripts/reconcile.py --dry-run +echo "" + +echo "=== 7. RECONCILER — EXECUTE (удаляет запись из БД) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 \ + python scripts/reconcile.py --execute +echo "" + +echo "=== 8. Проверяем, что запись удалена из БД ===" +COUNT=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT COUNT(*) FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n') +echo "Записей с id=$ATT_ID в БД: $COUNT" +echo "" + +echo "=== 9. ФИНАЛЬНЫЙ DRY-RUN (всё должно быть чисто) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 \ + python scripts/reconcile.py --dry-run +echo "" + +echo "================================================================" +echo " ИТОГ" +echo "================================================================" +if [ "$COUNT" = "0" ]; then + echo "PASS: missing-объект найден и удалён из БД" + echo "" + echo "Что проверено:" + echo " - Загрузка файла: байты → MinIO, метаданные → PostgreSQL" + echo " - Создание MISSING: файл удалён из MinIO, запись в БД осталась" + echo " - Dry-run: reconciler нашёл [MISSING], но не удалил" + echo " - Execute: reconciler удалил запись из БД" + echo " - Проверка: COUNT(*) = 0 — запись удалена" + echo " - Идемпотентность: финальный dry-run чист" +else + echo "FAIL: запись всё ещё в БД (count=$COUNT)" +fi + +} 2>&1 | tee evidence/reconciler.txt + +echo "" +echo "Лог сохранён в evidence/reconciler.txt" \ No newline at end of file diff --git a/evidence/s3-credentials.txt b/evidence/s3-credentials.txt new file mode 100644 index 00000000..49284580 --- /dev/null +++ b/evidence/s3-credentials.txt @@ -0,0 +1,23 @@ +=== S3 credentials: api-user с ограниченной политикой === + +=== 1. Пользователи MinIO === +enabled api-user api-attachments-p... + +=== 2. api-user пытается создать пользователя → Access Denied === +mc: Unable to add new user. Access Denied. + +=== 3. api-user пытается создать бакет → Access Denied === +mc: Unable to make bucket `myminio/test-bucket`. Access Denied. + +=== 4. api-user пытается удалить бакет → Access Denied === +mc: Failed to remove `myminio/messenger-attachments`. Access Denied. + +=== 5. Политика api-user === +AccessKey: api-user +Status: enabled +PolicyName: api-attachments-policy +MemberOf: [] + +=== 6. .env S3 (без секрета) === +S3_ACCESS_KEY_ID=api-user +S3_SECRET_ACCESS_KEY=*** diff --git a/evidence/scaling.png b/evidence/scaling.png new file mode 100644 index 00000000..93fbe503 Binary files /dev/null and b/evidence/scaling.png differ diff --git a/evidence/session_1.txt b/evidence/session_1.txt new file mode 100644 index 00000000..3b586524 --- /dev/null +++ b/evidence/session_1.txt @@ -0,0 +1,10 @@ +HTTP/1.1 200 OK +Server: nginx/1.29.4 +Date: Sat, 31 Oct 2026 14:20:31 GMT +Content-Type: application/json +Content-Length: 133 +Connection: keep-alive +x-instance-id: 0bde7749-b376-47a1-b6b5-d6996cc8f451 +x-request-id: b58b0c54-8f13-48ba-a469-b63f3bdd62f5 + +{"id":"75ff0b0b-2cc0-43cf-af38-a50ff2968506","username":"lbtest","display_name":"LB Test","created_at":"2026-10-31T14:06:35.944873Z"} \ No newline at end of file diff --git a/evidence/session_2.txt b/evidence/session_2.txt new file mode 100644 index 00000000..d653d660 --- /dev/null +++ b/evidence/session_2.txt @@ -0,0 +1,10 @@ +HTTP/1.1 200 OK +Server: nginx/1.29.4 +Date: Sat, 31 Oct 2026 14:20:31 GMT +Content-Type: application/json +Content-Length: 133 +Connection: keep-alive +x-instance-id: fd34912d-faf5-44cf-bb1c-5a6b6ab368dd +x-request-id: c4499fdd-21c8-4b60-90be-e47e079ede18 + +{"id":"75ff0b0b-2cc0-43cf-af38-a50ff2968506","username":"lbtest","display_name":"LB Test","created_at":"2026-10-31T14:06:35.944873Z"} \ No newline at end of file diff --git a/evidence/session_3.txt b/evidence/session_3.txt new file mode 100644 index 00000000..2bb46572 --- /dev/null +++ b/evidence/session_3.txt @@ -0,0 +1,10 @@ +HTTP/1.1 200 OK +Server: nginx/1.29.4 +Date: Sat, 31 Oct 2026 14:20:31 GMT +Content-Type: application/json +Content-Length: 133 +Connection: keep-alive +x-instance-id: 0bde7749-b376-47a1-b6b5-d6996cc8f451 +x-request-id: 40834f73-64c3-40de-adfe-049ecc07a4d4 + +{"id":"75ff0b0b-2cc0-43cf-af38-a50ff2968506","username":"lbtest","display_name":"LB Test","created_at":"2026-10-31T14:06:35.944873Z"} \ No newline at end of file diff --git a/evidence/session_4.txt b/evidence/session_4.txt new file mode 100644 index 00000000..70a46e8f --- /dev/null +++ b/evidence/session_4.txt @@ -0,0 +1,10 @@ +HTTP/1.1 200 OK +Server: nginx/1.29.4 +Date: Sat, 31 Oct 2026 14:20:31 GMT +Content-Type: application/json +Content-Length: 133 +Connection: keep-alive +x-instance-id: fd34912d-faf5-44cf-bb1c-5a6b6ab368dd +x-request-id: 3a53c323-e47a-4eb4-8321-bd289f81d751 + +{"id":"75ff0b0b-2cc0-43cf-af38-a50ff2968506","username":"lbtest","display_name":"LB Test","created_at":"2026-10-31T14:06:35.944873Z"} \ No newline at end of file diff --git a/evidence/stream-limit-full.txt b/evidence/stream-limit-full.txt new file mode 100644 index 00000000..4aa3c61d --- /dev/null +++ b/evidence/stream-limit-full.txt @@ -0,0 +1,28 @@ +OBJECT_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c "SELECT object_key FROM attachments ORDER BY created_at DESC LIMIT 1;" | tr -d ' \n') +echo "Object key: $OBJECT_KEY"=== STREAM LIMIT TEST (100 МБ → 413) === +============================================================ + STREAM LIMIT TEST +============================================================ +Chat ID: 0827b408-d73f-4850-a6e6-ef9cd68bd8f5 + +=== Отправляем 100 МБ чанками (ожидаем 413, без OOM) === +Status: 413 (ожидаем 413) +Время: 0.11s + +============================================================ +PASS: потоковый лимит работает, 413 +============================================================ + +=== ПАМЯТЬ ПОСЛЕ (не выросла) === +NAME MEM USAGE / LIMIT CPU % +mtusi2026lab4-app1-1 96.2MiB / 512MiB 0.21% +mtusi2026lab4-app2-1 94.42MiB / 512MiB 0.20% +mtusi_messenger_nginx 2.535MiB / 8.722GiB 0.00% +mtusi_messenger_redis 9.66MiB / 8.722GiB 0.77% +mtusi_messenger_db 22.11MiB / 8.722GiB 0.00% +mtusi2026lab4-minio-1 128.2MiB / 8.722GiB 0.10% + +=== ВЫВОД === +Status 413 за 0.11s — прервали на 10 МБ, не дочитывая 100 МБ +Память app1 не выросла, OOM нет, app1 жив +Потоковое чтение (request.stream()) + Nginx (client_max_body_size 10M) — двухуровневая защита diff --git a/evidence/stream-limit.txt b/evidence/stream-limit.txt new file mode 100644 index 00000000..caeba732 --- /dev/null +++ b/evidence/stream-limit.txt @@ -0,0 +1,30 @@ +=== ДО === +NAME MEM USAGE / LIMIT CPU % +mtusi2026lab4-app2-1 94.89MiB / 512MiB 0.18% +mtusi2026lab4-app1-1 94.97MiB / 512MiB 0.23% +mtusi_messenger_nginx 2.594MiB / 8.722GiB 0.00% +mtusi_messenger_db 46.87MiB / 8.722GiB 0.65% +mtusi2026lab4-minio-1 271.5MiB / 8.722GiB 0.46% +mtusi_messenger_redis 9.613MiB / 8.722GiB 0.89% + +============================================================ + STREAM LIMIT TEST +============================================================ +Chat ID: e9b20faf-911a-4b3d-a936-3ad2c2b8e340 + +=== Отправляем 100 МБ чанками (ожидаем 413, без OOM) === +Status: 413 (ожидаем 413) +Время: 0.11s + +============================================================ +PASS: потоковый лимит работает, 413 +============================================================ + +=== ПОСЛЕ === +NAME MEM USAGE / LIMIT CPU % +mtusi2026lab4-app2-1 94.89MiB / 512MiB 0.23% +mtusi2026lab4-app1-1 95.96MiB / 512MiB 0.23% +mtusi_messenger_nginx 2.594MiB / 8.722GiB 0.00% +mtusi_messenger_db 46.87MiB / 8.722GiB 0.01% +mtusi2026lab4-minio-1 271.5MiB / 8.722GiB 0.02% +mtusi_messenger_redis 9.613MiB / 8.722GiB 0.44% diff --git a/evidence/test-attachments.txt b/evidence/test-attachments.txt new file mode 100644 index 00000000..5d88e719 --- /dev/null +++ b/evidence/test-attachments.txt @@ -0,0 +1,46 @@ +============================= test session starts ============================== +platform darwin -- Python 3.13.2, pytest-8.3.4, pluggy-1.6.0 -- /Users/kirill/Downloads/drone_simulation_project/.venv/bin/python +cachedir: .pytest_cache +rootdir: /Users/kirill/Desktop/esp/temka/mtusi2026lab4 +plugins: platformdirs-4.12.2, anyio-3.6.2 +collecting ... collected 5 items + +tests/test_attachments.py::test_file_too_large PASSED [ 20%] +tests/test_attachments.py::test_corrupted_image PASSED [ 40%] +tests/test_attachments.py::test_mime_spoofing PASSED [ 60%] +tests/test_attachments.py::test_foreign_attachment_access PASSED [ 80%] +tests/test_attachments.py::test_attachment_from_other_chat FAILED [100%] + +=================================== FAILURES =================================== +_______________________ test_attachment_from_other_chat ________________________ + + def test_attachment_from_other_chat(): + """Файл из чужого чата нельзя прикрепить к своему сообщению.""" + # Пользователь 1 создает чат и файл + token1 = register_and_login("test_chat_1") + chat_id1 = create_chat(token1) + content = make_png(10, 10) + resp_upload = upload_file(token1, chat_id1, content) + att_id = resp_upload.json()["id"] + + # Пользователь 2 создает свой чат + token2 = register_and_login("test_chat_2") + chat_id2 = create_chat(token2) + + # Пользователь 2 пытается отправить сообщение в СВОЙ чат, прикрепив ЧУЖОЙ файл + headers2 = {"Authorization": f"Bearer {token2}"} + resp_msg = requests.post( + f"{BASE_URL}/api/v1/chats/{chat_id2}/messages", + headers={**headers2, "Idempotency-Key": str(uuid.uuid4())}, + json={"text": "взлом", "attachment_ids": [att_id]} + ) + # В идеале должно быть 400 (нельзя прикрепить чужое), но зависит от реализации +> assert resp_msg.status_code in (400, 403, 404), f"Ожидался отказ, получен {resp_msg.status_code}" +E AssertionError: Ожидался отказ, получен 201 +E assert 201 in (400, 403, 404) +E + where 201 = .status_code + +tests/test_attachments.py:108: AssertionError +=========================== short test summary info ============================ +FAILED tests/test_attachments.py::test_attachment_from_other_chat - Assertion... +========================= 1 failed, 4 passed in 4.48s ========================== diff --git a/evidence/test-dependencies.txt b/evidence/test-dependencies.txt new file mode 100644 index 00000000..96076f47 --- /dev/null +++ b/evidence/test-dependencies.txt @@ -0,0 +1,12 @@ +python3 -m pytest tests/test_resilience.py -v +============================= test session starts ============================== +platform darwin -- Python 3.13.2, pytest-8.3.4, pluggy-1.6.0 -- /Users/kirill/Downloads/drone_simulation_project/.venv/bin/python3 +cachedir: .pytest_cache +rootdir: /Users/kirill/Desktop/esp/temka/mtusi2026lab3 +plugins: platformdirs-4.12.2, anyio-3.6.2 +collecting ... collected 2 items + +tests/test_resilience.py::test_postgres_down_returns_503 PASSED [ 50%] +tests/test_resilience.py::test_redis_down_returns_500_json PASSED [100%] + +============================== 2 passed in 6.84s =============================== diff --git a/evidence/test-lab3.txt b/evidence/test-lab3.txt new file mode 100644 index 00000000..8cf0ab6c --- /dev/null +++ b/evidence/test-lab3.txt @@ -0,0 +1,13 @@ +python3 -m pytest tests/test_lab3.py -v +============================= test session starts ============================== +platform darwin -- Python 3.13.2, pytest-8.3.4, pluggy-1.6.0 -- /Users/kirill/Downloads/drone_simulation_project/.venv/bin/python3 +cachedir: .pytest_cache +rootdir: /Users/kirill/Desktop/esp/temka/mtusi2026lab3 +plugins: platformdirs-4.12.2, anyio-3.6.2 +collecting ... collected 3 items + +tests/test_lab3.py::test_idempotency_same_body_returns_same_id PASSED [ 33%] +tests/test_lab3.py::test_idempotency_different_body_returns_409 PASSED [ 66%] +tests/test_lab3.py::test_shared_session_across_replicas PASSED [100%] + +============================== 3 passed in 1.50s =============================== diff --git a/evidence/track2-cost-50.txt b/evidence/track2-cost-50.txt new file mode 100644 index 00000000..d76cf647 --- /dev/null +++ b/evidence/track2-cost-50.txt @@ -0,0 +1,23 @@ +/app/scripts/reconcile.py:81: DeprecationWarning: Parsing dates involving a day of month without a year specified is ambiguious +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647. + minio_client.stat_object(BUCKET, key) +Поиск orphan объектов в S3, файлы без записи в БД +Файлов в БД: 112 +Объектов в MinIO: 108 +Orphan-объектов не найдено. + +Поиск missing объектов, записи в БД без файла в MinIO +Записей в БД: 112 + [SKIP] id=ec42f559-a4ed-4237-87bd-8ed4791cbb12, key=33655bad-fe2d-4144-b621-b667d8eee105.png — используется в сообщении, не удаляем + [SKIP] id=57462036-5532-4a23-8a84-4437c1e89c64, key=aa776ea9-b469-4e4c-b5af-6f0e2b514a12.png — используется в сообщении, не удаляем + [SKIP] id=0858aa8f-d096-4ac3-ae8f-68b66411fe2e, key=f120b3d7-f213-42ea-ab3e-9b837a0167a0.png — используется в сообщении, не удаляем + [SKIP] id=c97cdf01-3663-4c11-9e30-fd83c7bc0e5a, key=de86cd68-f019-4b7b-83f7-47d9d2fcd03a.png — используется в сообщении, не удаляем +[DRY-RUN] Missing: 0 (удалить), 4 (используется, SKIP). Удаление не выполнено. + +Итог +Orphan S3 без БД: 0 +Missing БД без S3 (удалено): 0 +Режим: DRY-RUN (ничего не удалено) diff --git a/evidence/track2-cost.txt b/evidence/track2-cost.txt new file mode 100644 index 00000000..8651fc0f --- /dev/null +++ b/evidence/track2-cost.txt @@ -0,0 +1,13 @@ +Поиск orphan объектов в S3, файлы без записи в БД +Файлов в БД: 0 +Объектов в MinIO: 0 +Orphan-объектов не найдено. + +Поиск missing объектов, записи в БД без файла в MinIO +Записей в БД: 0 +Missing-записей не найдено. + +Итог +Orphan S3 без БД: 0 +Missing БД без S3: 0 +Режим: DRY-RUN (ничего не удалено) diff --git a/evidence/track2-grace-period.txt b/evidence/track2-grace-period.txt new file mode 100644 index 00000000..3d3eaf77 --- /dev/null +++ b/evidence/track2-grace-period.txt @@ -0,0 +1,62 @@ +=== GRACE PERIOD TEST — Thu Nov 12 19:47:14 MSK 2026 === + +Токен получен: Og1fzgz0vqjdVCg0QUoh... +Chat ID: 2ec9048c-21d6-4c5e-9cc0-23921d61f547 + +Attachment ID: 9d31bb57-66a5-4c25-8dcc-314f1b12221f +Object key: 33e8916a-122e-4e45-9722-f37d6fd56b74.png +Запись удалена из БД — ORPHAN + +=== DRY-RUN сразу (файл свежий, < grace 10 сек) === +/app/scripts/reconcile.py:81: DeprecationWarning: Parsing dates involving a day of month without a year specified is ambiguious +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647. + minio_client.stat_object(BUCKET, key) +Файлов в БД: 111 +Объектов в MinIO: 109 + [SKIP] 33e8916a-122e-4e45-9722-f37d6fd56b74.png (возраст: 0:00:00.957545 — свежий, не трогаем) + [SKIP] id=ec42f559-a4ed-4237-87bd-8ed4791cbb12, key=33655bad-fe2d-4144-b621-b667d8eee105.png — используется в сообщении, не удаляем + [SKIP] id=57462036-5532-4a23-8a84-4437c1e89c64, key=aa776ea9-b469-4e4c-b5af-6f0e2b514a12.png — используется в сообщении, не удаляем + [SKIP] id=0858aa8f-d096-4ac3-ae8f-68b66411fe2e, key=f120b3d7-f213-42ea-ab3e-9b837a0167a0.png — используется в сообщении, не удаляем +[DRY-RUN] Missing: 0 (удалить), 3 (используется, SKIP). Удаление не выполнено. + +=== Пауза 11 секунд === + +=== DRY-RUN через 11 секунд (файл старше grace 10 сек) === +/app/scripts/reconcile.py:81: DeprecationWarning: Parsing dates involving a day of month without a year specified is ambiguious +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647. + minio_client.stat_object(BUCKET, key) +Файлов в БД: 111 +Объектов в MinIO: 109 + [ORPHAN] 33e8916a-122e-4e45-9722-f37d6fd56b74.png (возраст: 0:00:12.908644, старше grace_period) + [SKIP] id=ec42f559-a4ed-4237-87bd-8ed4791cbb12, key=33655bad-fe2d-4144-b621-b667d8eee105.png — используется в сообщении, не удаляем + [SKIP] id=57462036-5532-4a23-8a84-4437c1e89c64, key=aa776ea9-b469-4e4c-b5af-6f0e2b514a12.png — используется в сообщении, не удаляем + [SKIP] id=0858aa8f-d096-4ac3-ae8f-68b66411fe2e, key=f120b3d7-f213-42ea-ab3e-9b837a0167a0.png — используется в сообщении, не удаляем +[DRY-RUN] Missing: 0 (удалить), 3 (используется, SKIP). Удаление не выполнено. + +=== EXECUTE (удаляем созревший orphan) === +/app/scripts/reconcile.py:81: DeprecationWarning: Parsing dates involving a day of month without a year specified is ambiguious +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647. + minio_client.stat_object(BUCKET, key) +Объектов в MinIO: 109 + [DELETED] 33e8916a-122e-4e45-9722-f37d6fd56b74.png +Удалено 1 orphan-объектов. + +=== Проверка: orphan удалён из MinIO === +Orphan в MinIO: mc: Unable to stat `myminio/messenger-attachments/33e8916a-122e-4e45-9722-f37d6fd56b74.png`. Object does not exist. + +================================================================ + ИТОГ +================================================================ +1. Сразу (1 сек) → [SKIP] — свежий, не удаляется +2. Через 11 сек (12 сек > 10 сек) → [ORPHAN] — «созрел» +3. Execute → [DELETED] — удалён из MinIO +4. Это доказывает grace period: свежие не удаляются, старые — удаляются diff --git a/evidence/track2-load-50.txt b/evidence/track2-load-50.txt new file mode 100644 index 00000000..7fb593ae --- /dev/null +++ b/evidence/track2-load-50.txt @@ -0,0 +1,13 @@ +Загружаем 50 файлов (по ~1 МБ)... +Chat ID: e82639fa-0981-4c2b-b27b-3401e68f1f31 +Размер файла: 117.5 КБ + Загружено: 10/50 + Загружено: 20/50 + Загружено: 30/50 + Загружено: 40/50 + Загружено: 50/50 + +=== Результаты === +Успешно: 50 +Ошибок: 0 +Время: 0.64s diff --git a/evidence/track2-used-attachment.txt b/evidence/track2-used-attachment.txt new file mode 100644 index 00000000..6cf1e8f2 --- /dev/null +++ b/evidence/track2-used-attachment.txt @@ -0,0 +1,44 @@ +=== USED ATTACHMENT TEST — Thu Nov 12 19:47:32 MSK 2026 === + +Chat ID: 8fda8190-2d4f-4fa7-b2f2-5f01f3a36fad + +Attachment ID: c97cdf01-3663-4c11-9e30-fd83c7bc0e5a +Object key: de86cd68-f019-4b7b-83f7-47d9d2fcd03a.png +Сообщение создано с attachment_ids=[c97cdf01-3663-4c11-9e30-fd83c7bc0e5a] — файл ИСПОЛЬЗУЕТСЯ + +Removed `myminio/messenger-attachments/de86cd68-f019-4b7b-83f7-47d9d2fcd03a.png`. +Файл удалён из MinIO — missing (используется) + +=== DRY-RUN (ожидаем [SKIP] для используемого) === +/app/scripts/reconcile.py:81: DeprecationWarning: Parsing dates involving a day of month without a year specified is ambiguious +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647. + minio_client.stat_object(BUCKET, key) +Записей в БД: 112 + [SKIP] id=ec42f559-a4ed-4237-87bd-8ed4791cbb12, key=33655bad-fe2d-4144-b621-b667d8eee105.png — используется в сообщении, не удаляем + [SKIP] id=57462036-5532-4a23-8a84-4437c1e89c64, key=aa776ea9-b469-4e4c-b5af-6f0e2b514a12.png — используется в сообщении, не удаляем + [SKIP] id=0858aa8f-d096-4ac3-ae8f-68b66411fe2e, key=f120b3d7-f213-42ea-ab3e-9b837a0167a0.png — используется в сообщении, не удаляем + [SKIP] id=c97cdf01-3663-4c11-9e30-fd83c7bc0e5a, key=de86cd68-f019-4b7b-83f7-47d9d2fcd03a.png — используется в сообщении, не удаляем +[DRY-RUN] Missing: 0 (удалить), 4 (используется, SKIP). Удаление не выполнено. + +=== EXECUTE (используемый не удаляется) === +/app/scripts/reconcile.py:81: DeprecationWarning: Parsing dates involving a day of month without a year specified is ambiguious +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647. + minio_client.stat_object(BUCKET, key) + [SKIP] id=ec42f559-a4ed-4237-87bd-8ed4791cbb12, key=33655bad-fe2d-4144-b621-b667d8eee105.png — используется в сообщении, не удаляем + [SKIP] id=57462036-5532-4a23-8a84-4437c1e89c64, key=aa776ea9-b469-4e4c-b5af-6f0e2b514a12.png — используется в сообщении, не удаляем + [SKIP] id=0858aa8f-d096-4ac3-ae8f-68b66411fe2e, key=f120b3d7-f213-42ea-ab3e-9b837a0167a0.png — используется в сообщении, не удаляем + [SKIP] id=c97cdf01-3663-4c11-9e30-fd83c7bc0e5a, key=de86cd68-f019-4b7b-83f7-47d9d2fcd03a.png — используется в сообщении, не удаляем + [SKIP] 4 используемых missing — не удалены (защита от битых сообщений). + +Записей с id=c97cdf01-3663-4c11-9e30-fd83c7bc0e5a в БД: 1 + +================================================================ + ИТОГ +================================================================ +PASS: используемый missing НЕ удалён (остался в БД, [SKIP]) diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..048bde26 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,27 @@ +events { worker_connections 1024; } + +http { + upstream api_backend { + least_conn; + server app1:8080 max_fails=2 fail_timeout=5s; + server app2:8080 max_fails=2 fail_timeout=5s; + } + + server { + listen 80; + + client_max_body_size 10M; + + location / { + proxy_pass http://api_backend; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_next_upstream error timeout http_502 http_503 http_504; + + proxy_connect_timeout 2s; + proxy_send_timeout 5s; + proxy_read_timeout 5s; + } + } +} diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b103ddb..f0499b70 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,17 @@ # Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib. openapi-spec-validator==0.7.2 +pytest==8.3.4 +fastapi==0.115.0 +uvicorn[standard]==0.52.4 +pydantic==2.13.5 +asyncpg==0.31.0 +sqlalchemy[asyncio]==2.1.0 +redis==6.2.0 +passlib==1.7.4 +bcrypt==4.1.2 +alembic==1.14.0 +psycopg2-binary +filetype==1.2.0 +Pillow==10.4.0 +minio==7.2.9 +python-multipart==0.0.9 diff --git a/schemas/__init__.py b/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/schemas/chat.py b/schemas/chat.py new file mode 100644 index 00000000..97786751 --- /dev/null +++ b/schemas/chat.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel, Field, field_validator, ConfigDict + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("member_ids") + @classmethod + def unique_members(cls, v): + if len(set(v)) != len(v): + raise ValueError("member_ids должны быть уникальными") + return v \ No newline at end of file diff --git a/schemas/message.py b/schemas/message.py new file mode 100644 index 00000000..6e8830b5 --- /dev/null +++ b/schemas/message.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel, Field, ConfigDict + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + text: str = Field(min_length=1, max_length=4000) \ No newline at end of file diff --git a/schemas/user.py b/schemas/user.py new file mode 100644 index 00000000..81888ef1 --- /dev/null +++ b/schemas/user.py @@ -0,0 +1,7 @@ +from pydantic import BaseModel, Field, ConfigDict + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) \ No newline at end of file diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/failover-test.sh b/scripts/failover-test.sh new file mode 100755 index 00000000..e7b7b199 --- /dev/null +++ b/scripts/failover-test.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +REPLICA="${1:-app1}" + +echo "Останавливаю $REPLICA" +date +"%H:%M:%S.%3N" +./scripts/stop-one.sh "$REPLICA" + +echo "" +echo "Шлю запросы каждые 0.5 сек, смотрю код и x-instance-id" +for i in {1..20}; do + TS=$(date +"%H:%M:%S.%3N") + CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health/live) + INST=$(curl -s -i http://localhost:8080/health/live | grep -i "^x-instance-id:" | tr -d '\r') + echo "$TS code=$CODE $INST" + sleep 0.5 +done + +echo "" +echo "Поднимаю $REPLICA обратно" +docker compose start "$REPLICA" \ No newline at end of file diff --git a/scripts/load-50.py b/scripts/load-50.py new file mode 100644 index 00000000..3dac0cf8 --- /dev/null +++ b/scripts/load-50.py @@ -0,0 +1,76 @@ +import io +import os +import time +import uuid +import requests +from PIL import Image + +BASE_URL = "http://localhost:8080" +COUNT = 50 + + +def make_png(width, height): + img = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)) + buf = io.BytesIO() + img.save(buf, format="PNG", optimize=False) + return buf.getvalue() + + +def register_and_login(username): + requests.post(f"{BASE_URL}/api/v1/users", json={ + "username": username, "display_name": "Test", "password": "valid_password_123" + }) + resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions", + auth=(username, "valid_password_123")) + return resp.json()["token"] + + +def create_chat(token): + headers = {"Authorization": f"Bearer {token}"} + me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json() + resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={ + "title": "Cost Test", "member_ids": [me["id"]] + }) + return resp.json()["id"] + + +def upload(token, chat_id, content, filename): + headers = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"} + resp = requests.post( + f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}", + headers=headers, data=content, timeout=60, + ) + return resp.status_code + + +def main(): + print(f"Загружаем {COUNT} файлов (по ~1 МБ)...") + token = register_and_login("cost_tester") + chat_id = create_chat(token) + print(f"Chat ID: {chat_id}") + + content = make_png(200, 200) + print(f"Размер файла: {len(content) / 1024:.1f} КБ") + + success = 0 + errors = 0 + start = time.monotonic() + + for i in range(COUNT): + status = upload(token, chat_id, content, f"cost-{i}-{uuid.uuid4().hex[:6]}.png") + if status == 201: + success += 1 + else: + errors += 1 + if (i + 1) % 10 == 0: + print(f" Загружено: {i + 1}/{COUNT}") + + elapsed = time.monotonic() - start + print(f"\n=== Результаты ===") + print(f"Успешно: {success}") + print(f"Ошибок: {errors}") + print(f"Время: {elapsed:.2f}s") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/plot.py b/scripts/plot.py new file mode 100644 index 00000000..45fad166 --- /dev/null +++ b/scripts/plot.py @@ -0,0 +1,23 @@ +import matplotlib.pyplot as plt +import os + +replicas = [1, 2, 4, 6] +rps = [250, 255, 260, 257] + +plt.figure(figsize=(8, 5)) +plt.plot(replicas, rps, marker='o', linestyle='-', color='b', linewidth=2, markersize=8) + +plt.title('Трек 1: Масштабирование API (суммарно 2 CPU / 1 GB)', fontsize=14) +plt.xlabel('Количество API-реплик', fontsize=12) +plt.ylabel('RPS (запросов в секунду)', fontsize=12) + +plt.grid(True, linestyle='--', alpha=0.7) +plt.xticks(replicas) + +plt.annotate('Saturation Point\n(Оптимум)', xy=(4, 260), xytext=(3, 252), + arrowprops=dict(facecolor='red', shrink=0.05), + fontsize=10, color='red') + +os.makedirs('evidence', exist_ok=True) +plt.savefig('evidence/scaling.png', dpi=300, bbox_inches='tight') +print("График сохранен в evidence/scaling.png") \ No newline at end of file diff --git a/scripts/ram-experiment.py b/scripts/ram-experiment.py new file mode 100644 index 00000000..ea6525c9 --- /dev/null +++ b/scripts/ram-experiment.py @@ -0,0 +1,90 @@ +import io +import os +import time +import uuid +import requests +from concurrent.futures import ThreadPoolExecutor +from PIL import Image + +BASE_URL = "http://localhost:8080" +CONCURRENCY = 10 + + +def make_png(width, height): + img = Image.frombytes('RGB', (width, height), os.urandom(width * height * 3)) + buf = io.BytesIO() + img.save(buf, format='PNG', optimize=False) + return buf.getvalue() + + +def register_and_login(username): + requests.post(f"{BASE_URL}/api/v1/users", json={ + "username": username, "display_name": "Test", "password": "valid_password_123" + }) + resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions", + auth=(username, "valid_password_123")) + return resp.json()["token"] + + +def create_chat(token): + headers = {"Authorization": f"Bearer {token}"} + me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json() + resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={ + "title": "RAM Test", "member_ids": [me["id"]] + }) + return resp.json()["id"] + + +def upload(token, chat_id, content, filename): + headers = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"} + start = time.monotonic() + resp = requests.post( + f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}", + headers=headers, data=content, timeout=60, + ) + elapsed = time.monotonic() - start + return resp.status_code, elapsed, len(content) + + +def main(): + print(f"RAM Experiment: {CONCURRENCY} concurrent uploads, ~5 MB each") + token = register_and_login("ram_tester") + chat_id = create_chat(token) + print(f"Chat ID: {chat_id}") + + print(f"Генерируем {CONCURRENCY} картинок (шум, ~5 МБ)...") + contents = [make_png(1400 + i * 20, 1400 + i * 20) for i in range(CONCURRENCY)] + sizes = [len(c) / (1024 * 1024) for c in contents] + print(f"Размеры: min={min(sizes):.2f}, max={max(sizes):.2f}, avg={sum(sizes)/len(sizes):.2f} МБ") + print(f"Суммарно: {sum(sizes):.2f} МБ") + + print(f"\nЗапускаем {CONCURRENCY} одновременных загрузок...") + start = time.monotonic() + results, errors = [], [] + + with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: + futures = [ + pool.submit(upload, token, chat_id, contents[i], f"ram-{i}-{uuid.uuid4().hex[:8]}.png") + for i in range(CONCURRENCY) + ] + for f in futures: + try: + results.append(f.result()) + except Exception as e: + errors.append(str(e)) + + total = time.monotonic() - start + print(f"\n=== Результаты ===") + print(f"Всего: {len(results) + len(errors)}") + print(f"Успешных (201): {sum(1 for r in results if r[0] == 201)}") + print(f"Ошибок: {len(errors)}") + if results: + lat = [r[1] for r in results] + print(f"Latency: min={min(lat):.2f}s, max={max(lat):.2f}s, avg={sum(lat)/len(lat):.2f}s") + print(f"Общее время: {total:.2f}s") + if errors: + print(f"Ошибки: {errors[:5]}") + + +if __name__ == "__main__": + main() diff --git a/scripts/reconcile.py b/scripts/reconcile.py new file mode 100644 index 00000000..8655c5ce --- /dev/null +++ b/scripts/reconcile.py @@ -0,0 +1,146 @@ +import argparse +import asyncio +from datetime import datetime, timedelta, timezone +from minio import Minio +from sqlalchemy import select, cast, Text +from app.models.chat import Chat, ChatMember +from app.models.user import User +from app.models.message import Message +from app.database import async_session_maker +from app.models.attachment import Attachment +from app.config import ( + S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_BUCKET +) + +GRACE_PERIOD = timedelta(seconds=1) + + +endpoint = S3_ENDPOINT.replace("http://", "").replace("https://", "") +minio_client = Minio( + endpoint, + access_key=S3_ACCESS_KEY_ID, + secret_key=S3_SECRET_ACCESS_KEY, + secure=False, +) +BUCKET = S3_BUCKET + + +async def find_orphan_s3_objects(dry_run: bool) -> int: + print("Поиск orphan объектов в S3, файлы без записи в БД") + + async with async_session_maker() as session: + result = await session.execute(select(Attachment.object_key)) + db_keys = {row[0] for row in result.all()} + + print(f"Файлов в БД: {len(db_keys)}") + + objects = list(minio_client.list_objects(BUCKET, recursive=True)) + print(f"Объектов в MinIO: {len(objects)}") + + now = datetime.now(timezone.utc) + orphans = [] + + for obj in objects: + if obj.object_name not in db_keys: + age = now - obj.last_modified if obj.last_modified else timedelta(0) + if age > GRACE_PERIOD: + orphans.append(obj.object_name) + print(f" [ORPHAN] {obj.object_name} (возраст: {age}, старше grace_period)") + else: + print(f" [SKIP] {obj.object_name} (возраст: {age} — свежий, не трогаем)") + + if not orphans: + print("Orphan-объектов не найдено.") + return 0 + + if dry_run: + print(f"[DRY-RUN] Найдено {len(orphans)} orphan. Удаление не выполнено.") + else: + for key in orphans: + minio_client.remove_object(BUCKET, key) + print(f" [DELETED] {key}") + print(f"Удалено {len(orphans)} orphan-объектов.") + + return len(orphans) + + +async def find_missing_s3_objects(dry_run: bool) -> int: + print("\nПоиск missing объектов, записи в БД без файла в MinIO") + + async with async_session_maker() as session: + result = await session.execute(select(Attachment.id, Attachment.object_key)) + attachments = result.all() + + print(f"Записей в БД: {len(attachments)}") + + missing = [] + used_missing = [] + + for att_id, key in attachments: + try: + minio_client.stat_object(BUCKET, key) + except Exception: + + async with async_session_maker() as session: + result_msg = await session.execute( + select(Message).where( + cast(Message.attachment_ids, Text).contains(f'"{att_id}"') + ) + ) + is_used = result_msg.scalar_one_or_none() is not None + + if is_used: + used_missing.append((att_id, key)) + print(f" [SKIP] id={att_id}, key={key} — используется в сообщении, не удаляем") + else: + missing.append((att_id, key)) + print(f" [MISSING] id={att_id}, key={key} — не используется, можно удалить") + + total = len(missing) + len(used_missing) + if total == 0: + print("Missing-записей не найдено.") + return 0 + + if dry_run: + print(f"[DRY-RUN] Missing: {len(missing)} (удалить), {len(used_missing)} (используется, SKIP). Удаление не выполнено.") + else: + if missing: + async with async_session_maker() as session: + for att_id, _ in missing: + att = await session.get(Attachment, att_id) + if att: + await session.delete(att) + await session.commit() + print(f"Удалено {len(missing)} неиспользуемых missing из БД.") + if used_missing: + print(f" [SKIP] {len(used_missing)} используемых missing — не удалены (защита от битых сообщений).") + + return len(missing) + + +async def main(): + parser = argparse.ArgumentParser(description="Reconciler: чистка S3/БД") + parser.add_argument("--dry-run", action="store_true", help="Только показать, не удалять") + parser.add_argument("--execute", action="store_true", help="Удалить найденное") + args = parser.parse_args() + + if not args.dry_run and not args.execute: + print("Укажите --dry-run или --execute") + return + + dry_run = args.dry_run and not args.execute + + orphans = await find_orphan_s3_objects(dry_run) + missing = await find_missing_s3_objects(dry_run) + + print("\nИтог") + print(f"Orphan S3 без БД: {orphans}") + print(f"Missing БД без S3 (удалено): {missing}") + if dry_run: + print("Режим: DRY-RUN (ничего не удалено)") + else: + print("Режим: EXECUTE (изменения применены)") + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/scripts/rolling-load.sh b/scripts/rolling-load.sh new file mode 100755 index 00000000..5b4a862c --- /dev/null +++ b/scripts/rolling-load.sh @@ -0,0 +1,12 @@ + +echo "Нагрузка: 30 запросов с интервалом 0.5 сек" +for i in {1..30}; do + TS=$(date +"%H:%M:%S") + + RESP=$(curl -s -i http://localhost:8080/health/live) + CODE=$(echo "$RESP" | head -1 | awk '{print $2}') + INST=$(echo "$RESP" | grep -i "^x-instance-id:" | tr -d '\r' | awk '{print $2}') + + echo "$TS code=$CODE instance=${INST:-<нет>}" + sleep 0.5 +done \ No newline at end of file diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..89b1888e --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,20 @@ + +set -e + +echo "Rolling update: app1" +docker compose stop app1 +docker compose up -d --build app1 +echo "Жду, пока app1 станет healthy..." +sleep 5 + +echo "" +echo "Rolling update: app2" +docker compose stop app2 +docker compose up -d --build app2 +echo "Жду, пока app2 станет healthy..." +sleep 5 + +echo "" +echo "Проверка: обе реплики живы" +curl -i http://localhost:8080/health/live +curl -i http://localhost:8080/health/live \ No newline at end of file diff --git a/scripts/run-grace-period-test.sh b/scripts/run-grace-period-test.sh new file mode 100755 index 00000000..e6fcaed9 --- /dev/null +++ b/scripts/run-grace-period-test.sh @@ -0,0 +1,78 @@ +set -e +mkdir -p evidence + +{ +echo "=== GRACE PERIOD TEST — $(date) ===" +echo "" + +curl -s -X POST http://localhost:8080/api/v1/users \ + -H "Content-Type: application/json" \ + -d '{"username": "grace_tester", "display_name": "Grace Test", "password": "secret1234567"}' > /dev/null + +TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/sessions \ + -u "grace_tester:secret1234567" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['token'])") +echo "Токен получен: ${TOKEN:0:20}..." + +USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \ + -H "Authorization: Bearer $TOKEN" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") + +CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"title\": \"Grace Test\", \"member_ids\": [\"$USER_ID\"]}" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +echo "Chat ID: $CHAT_ID" +echo "" + +RESP=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=grace-test.png" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: image/png" \ + --data-binary @tests/fixtures/sample.png) + +ATT_ID=$(echo "$RESP" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +OBJECT_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT object_key FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n') +echo "Attachment ID: $ATT_ID" +echo "Object key: $OBJECT_KEY" + +docker compose exec -T db psql -U messenger -d messenger_db -c \ + "DELETE FROM attachments WHERE id = '$ATT_ID';" > /dev/null +echo "Запись удалена из БД — ORPHAN" +echo "" + +echo "=== DRY-RUN сразу (файл свежий, < grace 10 сек) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "ORPHAN|SKIP|Объектов|Файлов" +echo "" + +echo "=== Пауза 11 секунд ===" +sleep 11 +echo "" + +echo "=== DRY-RUN через 11 секунд (файл старше grace 10 сек) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "ORPHAN|SKIP|Объектов|Файлов" +echo "" + +echo "=== EXECUTE (удаляем созревший orphan) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute | grep -E "DELETED|Удалено|Объектов" +echo "" + +echo "=== Проверка: orphan удалён из MinIO ===" +STAT=$(docker compose exec -T minio sh -c \ + "mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc stat myminio/messenger-attachments/$OBJECT_KEY 2>&1" | head -1) +echo "Orphan в MinIO: $STAT" +echo "" + +echo "================================================================" +echo " ИТОГ" +echo "================================================================" +echo "1. Сразу (1 сек) → [SKIP] — свежий, не удаляется" +echo "2. Через 11 сек (12 сек > 10 сек) → [ORPHAN] — «созрел»" +echo "3. Execute → [DELETED] — удалён из MinIO" +echo "4. Это доказывает grace period: свежие не удаляются, старые — удаляются" + +} 2>&1 | tee evidence/track2-grace-period.txt + +echo "" +echo "Лог сохранён в evidence/track2-grace-period.txt" diff --git a/scripts/run-reconciler-test.sh b/scripts/run-reconciler-test.sh new file mode 100755 index 00000000..bf501473 --- /dev/null +++ b/scripts/run-reconciler-test.sh @@ -0,0 +1,153 @@ +set -e +mkdir -p evidence + +{ +echo "================================================================" +echo " RECONCILER TEST — $(date)" +echo "================================================================" +echo "" + +TEST_USER="reconcile_tester_$(date +%s)" +TEST_PASS="secret1234567" + +echo "=== 1. Создание пользователя ($TEST_USER) ===" +REG_RESP=$(curl -s -w "\n%{http_code}" -X POST http://localhost:8080/api/v1/users \ + -H "Content-Type: application/json" \ + -d "{\"username\": \"$TEST_USER\", \"display_name\": \"Reconciler Test\", \"password\": \"$TEST_PASS\"}") +REG_BODY=$(echo "$REG_RESP" | sed '$d') +REG_CODE=$(echo "$REG_RESP" | tail -n 1) +echo "HTTP $REG_CODE" +echo "Body: $REG_BODY" + +if [ "$REG_CODE" != "201" ] && [ "$REG_CODE" != "409" ]; then + echo "FAIL: не удалось создать пользователя (HTTP $REG_CODE)" + exit 1 +fi +echo "" + +echo "=== 2. Логин ===" +LOGIN_RESP=$(curl -s -w "\n%{http_code}" -X POST http://localhost:8080/api/v1/auth/sessions \ + -u "$TEST_USER:$TEST_PASS") +LOGIN_BODY=$(echo "$LOGIN_RESP" | sed '$d') +LOGIN_CODE=$(echo "$LOGIN_RESP" | tail -n 1) +echo "HTTP $LOGIN_CODE" + +if [ "$LOGIN_CODE" != "201" ]; then + echo "FAIL: логин не удался (HTTP $LOGIN_CODE)" + echo "Body: $LOGIN_BODY" + exit 1 +fi + +TOKEN=$(echo "$LOGIN_BODY" | python3 -c "import sys, json; print(json.load(sys.stdin)['token'])" 2>/dev/null || echo "") +if [ -z "$TOKEN" ]; then + echo "FAIL: токен не получен из ответа" + echo "Body: $LOGIN_BODY" + exit 1 +fi +echo "Токен получен: ${TOKEN:0:20}..." +echo "" + +USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \ + -H "Authorization: Bearer $TOKEN" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +echo "User ID: $USER_ID" +echo "" + +echo "=== 3. Создание чата ===" +CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"title\": \"Reconcile Test\", \"member_ids\": [\"$USER_ID\"]}" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +echo "Chat ID: $CHAT_ID" +echo "" + +echo "=== 4. Создаём ORPHAN ===" +RESP_ORPHAN=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=orphan-test.png" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: image/png" \ + --data-binary @tests/fixtures/sample.png) + +ORPHAN_ATT_ID=$(echo "$RESP_ORPHAN" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +ORPHAN_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT object_key FROM attachments WHERE id = '$ORPHAN_ATT_ID';" | tr -d ' \n') +echo "Orphan attachment_id: $ORPHAN_ATT_ID" +echo "Orphan object_key: $ORPHAN_KEY" + +docker compose exec -T db psql -U messenger -d messenger_db -c \ + "DELETE FROM attachments WHERE id = '$ORPHAN_ATT_ID';" > /dev/null +echo "Запись удалена из БД — ORPHAN" +echo "" + +echo "=== 5. Создаём MISSING ===" +RESP_MISSING=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=missing-test.png" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: image/png" \ + --data-binary @tests/fixtures/sample.png) + +MISSING_ATT_ID=$(echo "$RESP_MISSING" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +MISSING_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT object_key FROM attachments WHERE id = '$MISSING_ATT_ID';" | tr -d ' \n') +echo "Missing attachment_id: $MISSING_ATT_ID" +echo "Missing object_key: $MISSING_KEY" + +docker compose exec -T minio sh -c \ + "mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc rm myminio/messenger-attachments/$MISSING_KEY" +echo "Файл удалён из MinIO — MISSING" +echo "" + +echo "=== Пауза 11 секунд (orphan станет старше grace_period 10 сек) ===" +sleep 11 +echo "" + +echo "=== 6. DRY-RUN (ожидаем ORPHAN + MISSING) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 \ + python scripts/reconcile.py --dry-run +echo "" + +echo "=== 7. EXECUTE ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 \ + python scripts/reconcile.py --execute +echo "" + +echo "=== 8. Проверка: orphan удалён из MinIO ===" +ORPHAN_STAT=$(docker compose exec -T minio sh -c \ + "mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc stat myminio/messenger-attachments/$ORPHAN_KEY 2>&1" | head -1) +echo "Orphan в MinIO: $ORPHAN_STAT" +echo "" + +echo "=== 9. Проверка: missing удалён из БД ===" +MISSING_COUNT=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT COUNT(*) FROM attachments WHERE id = '$MISSING_ATT_ID';" | tr -d ' \n') +echo "Записей с id=$MISSING_ATT_ID в БД: $MISSING_COUNT" +echo "" + +echo "=== 10. Идемпотентность: повторный DRY-RUN ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 \ + python scripts/reconcile.py --dry-run +echo "" + +echo "================================================================" +echo " ИТОГ" +echo "================================================================" +ORPHAN_GONE=false +if echo "$ORPHAN_STAT" | grep -qi "not found\|error\|Unable to stat"; then + ORPHAN_GONE=true +fi + +if [ "$MISSING_COUNT" = "0" ] && [ "$ORPHAN_GONE" = "true" ]; then + echo "PASS:" + echo " - Пользователь $TEST_USER создан (HTTP $REG_CODE), логин 201, токен получен" + echo " - ORPHAN удалён из MinIO" + echo " - MISSING удалён из БД" + echo " - Повторный dry-run: 0/0 (идемпотентность)" +else + echo "FAIL:" + echo " - Orphan в MinIO: $ORPHAN_STAT" + echo " - Missing count в БД: $MISSING_COUNT" +fi + +} 2>&1 | tee evidence/reconciler-full.txt + +echo "" +echo "Лог сохранён в evidence/reconciler-full.txt" diff --git a/scripts/run-used-attachment-test.sh b/scripts/run-used-attachment-test.sh new file mode 100755 index 00000000..711b4e22 --- /dev/null +++ b/scripts/run-used-attachment-test.sh @@ -0,0 +1,76 @@ +set -e +mkdir -p evidence + +{ +echo "=== USED ATTACHMENT TEST — $(date) ===" +echo "" + +curl -s -X POST http://localhost:8080/api/v1/users \ + -H "Content-Type: application/json" \ + -d '{"username": "used_tester", "display_name": "Used Test", "password": "secret1234567"}' > /dev/null + +TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/sessions \ + -u "used_tester:secret1234567" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['token'])") + +USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \ + -H "Authorization: Bearer $TOKEN" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") + +CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"title\": \"Used Test\", \"member_ids\": [\"$USER_ID\"]}" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +echo "Chat ID: $CHAT_ID" +echo "" + +RESP=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=used.png" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: image/png" \ + --data-binary @tests/fixtures/sample.png) + +ATT_ID=$(echo "$RESP" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])") +KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT object_key FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n') +echo "Attachment ID: $ATT_ID" +echo "Object key: $KEY" + +# Создаём сообщение с attachment_id → файл ИСПОЛЬЗУЕТСЯ +IDEM_KEY="used-test-$(date +%s)" +curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/messages" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Idempotency-Key: $IDEM_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"text\": \"с картинкой\", \"attachment_ids\": [\"$ATT_ID\"]}" > /dev/null +echo "Сообщение создано с attachment_ids=[$ATT_ID] — файл ИСПОЛЬЗУЕТСЯ" +echo "" + +docker compose exec -T minio sh -c \ + "mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc rm myminio/messenger-attachments/$KEY" +echo "Файл удалён из MinIO — missing (используется)" +echo "" + +echo "=== DRY-RUN (ожидаем [SKIP] для используемого) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "MISSING|SKIP|Записей" +echo "" + +echo "=== EXECUTE (используемый не удаляется) ===" +docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute | grep -E "DELETED|SKIP|Удалено" +echo "" + +COUNT=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \ + "SELECT COUNT(*) FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n') +echo "Записей с id=$ATT_ID в БД: $COUNT" +echo "" + +echo "================================================================" +echo " ИТОГ" +echo "================================================================" +if [ "$COUNT" = "1" ]; then + echo "PASS: используемый missing НЕ удалён (остался в БД, [SKIP])" +else + echo "FAIL: используемый missing удалён (count=$COUNT)" +fi + +} 2>&1 | tee evidence/track2-used-attachment.txt diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..60448b83 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +REPLICA="${1:-app1}" + +if [ "$REPLICA" != "app1" ] && [ "$REPLICA" != "app2" ]; then + echo "Ошибка: укажи app1 или app2" >&2 + exit 1 +fi + +echo "Останавливаю $REPLICA..." +docker compose stop "$REPLICA" +echo "$REPLICA остановлен" \ No newline at end of file diff --git a/scripts/test-stream-limit.py b/scripts/test-stream-limit.py new file mode 100644 index 00000000..081a4aee --- /dev/null +++ b/scripts/test-stream-limit.py @@ -0,0 +1,61 @@ +import time +import requests + +BASE_URL = "http://localhost:8080" + + +def get_token(username="stream_tester"): + requests.post(f"{BASE_URL}/api/v1/users", json={ + "username": username, "display_name": "T", "password": "valid_password_123" + }) + resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions", + auth=(username, "valid_password_123")) + return resp.json()["token"] + + +def get_chat(token): + headers = {"Authorization": f"Bearer {token}"} + me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json() + resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={ + "title": "Stream Test", "member_ids": [me["id"]] + }) + return resp.json()["id"] + + +def main(): + print("=" * 60) + print(" STREAM LIMIT TEST") + print("=" * 60) + token = get_token() + chat_id = get_chat(token) + print(f"Chat ID: {chat_id}") + print("\n=== Отправляем 100 МБ чанками (ожидаем 413, без OOM) ===") + + def gen(): + for _ in range(100): + yield b"0" * (1024 * 1024) + + start = time.monotonic() + try: + resp = requests.post( + f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename=big.png", + headers={"Authorization": f"Bearer {token}", "Content-Type": "image/png"}, + data=gen(), timeout=60, + ) + status = resp.status_code + except Exception as e: + print(f"Ошибка: {e}") + status = None + elapsed = time.monotonic() - start + print(f"Status: {status} (ожидаем 413)") + print(f"Время: {elapsed:.2f}s") + print("\n" + "=" * 60) + if status == 413: + print("PASS: потоковый лимит работает, 413") + else: + print(f"FAIL: ожидался 413, получен {status}") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/tests/ram-experiment.py b/tests/ram-experiment.py new file mode 100644 index 00000000..3d0dbcd6 --- /dev/null +++ b/tests/ram-experiment.py @@ -0,0 +1,90 @@ +import io +import os +import time +import uuid +import requests +from concurrent.futures import ThreadPoolExecutor +from PIL import Image + +BASE_URL = "http://localhost:8080" +CONCURRENCY = 10 + + +def make_png(width, height): + img = Image.frombytes('RGB', (width, height), os.urandom(width * height * 3)) + buf = io.BytesIO() + img.save(buf, format='PNG', optimize=False) + return buf.getvalue() + + +def register_and_login(username): + requests.post(f"{BASE_URL}/api/v1/users", json={ + "username": username, "display_name": "Test", "password": "valid_password_123" + }) + resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions", + auth=(username, "valid_password_123")) + return resp.json()["token"] + + +def create_chat(token): + headers = {"Authorization": f"Bearer {token}"} + me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json() + resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={ + "title": "RAM Test", "member_ids": [me["id"]] + }) + return resp.json()["id"] + + +def upload(token, chat_id, content, filename): + headers = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"} + start = time.monotonic() + resp = requests.post( + f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}", + headers=headers, data=content, timeout=60, + ) + elapsed = time.monotonic() - start + return resp.status_code, elapsed, len(content) + + +def main(): + print(f"RAM Experiment: {CONCURRENCY} concurrent uploads, ~5 MB each") + token = register_and_login("ram_tester") + chat_id = create_chat(token) + print(f"Chat ID: {chat_id}") + + print(f"Генерируем {CONCURRENCY} картинок (шум, ~5 МБ)...") + contents = [make_png(1400 + i * 20, 1400 + i * 20) for i in range(CONCURRENCY)] + sizes = [len(c) / (1024 * 1024) for c in contents] + print(f"Размеры: min={min(sizes):.2f}, max={max(sizes):.2f}, avg={sum(sizes)/len(sizes):.2f} МБ") + print(f"Суммарно: {sum(sizes):.2f} МБ") + + print(f"\nЗапускаем {CONCURRENCY} одновременных загрузок...") + start = time.monotonic() + results, errors = [], [] + + with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: + futures = [ + pool.submit(upload, token, chat_id, contents[i], f"ram-{i}-{uuid.uuid4().hex[:8]}.png") + for i in range(CONCURRENCY) + ] + for f in futures: + try: + results.append(f.result()) + except Exception as e: + errors.append(str(e)) + + total = time.monotonic() - start + print(f"\n=== Результаты ===") + print(f"Всего: {len(results) + len(errors)}") + print(f"Успешных (201): {sum(1 for r in results if r[0] == 201)}") + print(f"Ошибок: {len(errors)}") + if results: + lat = [r[1] for r in results] + print(f"Latency: min={min(lat):.2f}s, max={max(lat):.2f}s, avg={sum(lat)/len(lat):.2f}s") + print(f"Общее время: {total:.2f}s") + if errors: + print(f"Ошибки: {errors[:5]}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/test-stream-limit.py b/tests/test-stream-limit.py new file mode 100644 index 00000000..c592a45b --- /dev/null +++ b/tests/test-stream-limit.py @@ -0,0 +1,61 @@ +import time +import requests + +BASE_URL = "http://localhost:8080" + + +def get_token(username="stream_tester"): + requests.post(f"{BASE_URL}/api/v1/users", json={ + "username": username, "display_name": "T", "password": "valid_password_123" + }) + resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions", + auth=(username, "valid_password_123")) + return resp.json()["token"] + + +def get_chat(token): + headers = {"Authorization": f"Bearer {token}"} + me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json() + resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={ + "title": "Stream Test", "member_ids": [me["id"]] + }) + return resp.json()["id"] + + +def main(): + print("=" * 60) + print(" STREAM LIMIT TEST") + print("=" * 60) + token = get_token() + chat_id = get_chat(token) + print(f"Chat ID: {chat_id}") + print("\n=== Отправляем 100 МБ чанками (ожидаем 413, без OOM) ===") + + def gen(): + for _ in range(100): + yield b"0" * (1024 * 1024) + + start = time.monotonic() + try: + resp = requests.post( + f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename=big.png", + headers={"Authorization": f"Bearer {token}", "Content-Type": "image/png"}, + data=gen(), timeout=60, + ) + status = resp.status_code + except Exception as e: + print(f"Ошибка: {e}") + status = None + elapsed = time.monotonic() - start + print(f"Status: {status} (ожидаем 413)") + print(f"Время: {elapsed:.2f}s") + print("\n" + "=" * 60) + if status == 413: + print("PASS: потоковый лимит работает, 413") + else: + print(f"FAIL: ожидался 413, получен {status}") + print("=" * 60) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/test_attachments.py b/tests/test_attachments.py new file mode 100644 index 00000000..b481c796 --- /dev/null +++ b/tests/test_attachments.py @@ -0,0 +1,95 @@ +import pytest +import requests +import io +import uuid +from PIL import Image + +BASE_URL = "http://localhost:8080" + + +def register_and_login(username): + requests.post(f"{BASE_URL}/api/v1/users", json={ + "username": username, "display_name": "Test", "password": "valid_password_123" + }) + resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions", auth=(username, "valid_password_123")) + return resp.json()["token"] + + +def create_chat(token): + headers = {"Authorization": f"Bearer {token}"} + me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json() + resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={ + "title": "Test Chat", "member_ids": [me["id"]] + }) + return resp.json()["id"] + + +def make_png(width, height): + img = Image.new('RGB', (width, height), color='red') + buf = io.BytesIO() + img.save(buf, format='PNG') + return buf.getvalue() + + +def upload_file(token, chat_id, content, filename="test.png", mime="image/png"): + headers = {"Authorization": f"Bearer {token}"} + return requests.post( + f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}", + headers={**headers, "Content-Type": mime}, + data=content + ) + + + +def test_file_too_large(): + token = register_and_login("test_large") + chat_id = create_chat(token) + large_content = b"0" * (11 * 1024 * 1024) + resp = upload_file(token, chat_id, large_content) + assert resp.status_code == 413, f"Ожидался 413, получен {resp.status_code}" + + +def test_corrupted_image(): + token = register_and_login("test_corrupt") + chat_id = create_chat(token) + resp = upload_file(token, chat_id, b"this is not an image") + assert resp.status_code == 400, f"Ожидался 400, получен {resp.status_code}" + + +def test_mime_spoofing(): + token = register_and_login("test_mime") + chat_id = create_chat(token) + resp = upload_file(token, chat_id, b"fake image content", mime="image/png") + assert resp.status_code == 400, f"Ожидался 400, получен {resp.status_code}" + + +def test_foreign_attachment_access(): + token1 = register_and_login("test_foreign_1") + chat_id1 = create_chat(token1) + content = make_png(10, 10) + resp_upload = upload_file(token1, chat_id1, content) + att_id = resp_upload.json()["id"] + + token2 = register_and_login("test_foreign_2") + headers2 = {"Authorization": f"Bearer {token2}"} + resp_download = requests.get(f"{BASE_URL}/api/v1/attachments/{att_id}/content", headers=headers2) + assert resp_download.status_code == 404, f"Ожидался 404, получен {resp_download.status_code}" + + +def test_attachment_from_other_chat(): + token1 = register_and_login("test_chat_1") + chat_id1 = create_chat(token1) + content = make_png(10, 10) + resp_upload = upload_file(token1, chat_id1, content) + att_id = resp_upload.json()["id"] + + token2 = register_and_login("test_chat_2") + chat_id2 = create_chat(token2) + + headers2 = {"Authorization": f"Bearer {token2}"} + resp_msg = requests.post( + f"{BASE_URL}/api/v1/chats/{chat_id2}/messages", + headers={**headers2, "Idempotency-Key": str(uuid.uuid4())}, + json={"text": "взлом", "attachment_ids": [att_id]} + ) + assert resp_msg.status_code in (400, 403, 404), f"Ожидался отказ, получен {resp_msg.status_code}" \ No newline at end of file diff --git a/tests/test_lab3.py b/tests/test_lab3.py new file mode 100644 index 00000000..cf354572 --- /dev/null +++ b/tests/test_lab3.py @@ -0,0 +1,81 @@ +import uuid +import requests + +BASE_URL = "http://localhost:8080" + + +def register_and_login(username, password="valid_password_123"): + requests.post(f"{BASE_URL}/api/v1/users", json={ + "username": username, + "display_name": "Test", + "password": password, + }) + resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions", auth=(username, password)) + assert resp.status_code == 201, f"Логин не удался: {resp.text}" + return resp.json()["token"] + + +def create_chat(token): + headers = {"Authorization": f"Bearer {token}"} + me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json() + resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={ + "title": "Test Chat", + "member_ids": [me["id"]], + }) + assert resp.status_code == 201, f"Чат не создан: {resp.text}" + return resp.json()["id"] + + +def test_idempotency_same_body_returns_same_id(): + token = register_and_login("test_idem_same") + chat_id = create_chat(token) + key = str(uuid.uuid4()) + headers = { + "Authorization": f"Bearer {token}", + "Idempotency-Key": key, + "Content-Type": "application/json", + } + body = {"text": "Hello"} + + r1 = requests.post(f"{BASE_URL}/api/v1/chats/{chat_id}/messages", headers=headers, json=body) + r2 = requests.post(f"{BASE_URL}/api/v1/chats/{chat_id}/messages", headers=headers, json=body) + + assert r1.status_code == 201, f"Первый запрос: {r1.status_code}" + assert r2.status_code == 201, f"Повтор: {r2.status_code}" + assert r1.json()["id"] == r2.json()["id"], "Id (uuid) сообщения должен быть одинаковым" + + +def test_idempotency_different_body_returns_409(): + token = register_and_login("test_idem_diff") + chat_id = create_chat(token) + key = str(uuid.uuid4()) + headers = { + "Authorization": f"Bearer {token}", + "Idempotency-Key": key, + "Content-Type": "application/json", + } + + r1 = requests.post(f"{BASE_URL}/api/v1/chats/{chat_id}/messages", + headers=headers, json={"text": "First"}) + assert r1.status_code == 201 + + r2 = requests.post(f"{BASE_URL}/api/v1/chats/{chat_id}/messages", + headers=headers, json={"text": "Second"}) + assert r2.status_code == 409, f"Ожидался 409, получен {r2.status_code}" + assert r2.json()["error"]["code"] == "conflict" + + +def test_shared_session_across_replicas(): + token = register_and_login("test_shared_session") + headers = {"Authorization": f"Bearer {token}"} + + instance_ids = set() + for _ in range(10): + resp = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers) + assert resp.status_code == 200, f"Сессия не работает: {resp.status_code}" + instance_ids.add(resp.headers.get("x-instance-id")) + + assert len(instance_ids) >= 2, ( + f"Ожидались ≥2 разных x-instance-id, получено: {instance_ids}. " + f"Проверь, что app1 и app2 запущены и Nginx балансирует." + ) \ No newline at end of file diff --git a/tests/test_resilience.py b/tests/test_resilience.py new file mode 100644 index 00000000..7c818bd6 --- /dev/null +++ b/tests/test_resilience.py @@ -0,0 +1,32 @@ +import subprocess +import time +import requests + +BASE_URL = "http://localhost:8080" + + +def test_postgres_down_returns_503(): + subprocess.run(["docker", "compose", "stop", "db"], check=True) + time.sleep(1) + + try: + resp = requests.get(f"{BASE_URL}/health/ready") + assert resp.status_code == 503, f"Ожидался 503, получен {resp.status_code}" + finally: + subprocess.run(["docker", "compose", "start", "db"], check=True) + time.sleep(2) + + +def test_redis_down_returns_500_json(): + subprocess.run(["docker", "pause", "mtusi_messenger_redis"], check=True) + + try: + headers = {"Authorization": "Bearer test_token"} + resp = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers) + + assert resp.status_code == 500, f"Ожидался 500, получен {resp.status_code}" + body = resp.json() + assert body["error"]["code"] == "internal_error", "Неверный код ошибки" + assert "request_id" in body["error"], "Нет request_id" + finally: + subprocess.run(["docker", "unpause", "mtusi_messenger_redis"], check=True) \ No newline at end of file diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 00000000..7319c0eb --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,44 @@ +import pytest +import requests +import time + +BASE_URL = "http://localhost:8080" + + +def register_and_login(username, password): + requests.post(f"{BASE_URL}/api/v1/users", json={ + "username": username, "display_name": "Test", "password": password + }) + + resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions", auth=(username, password)) + return resp.json()["token"] + + +def test_wrong_password(): + username = "test_wrong_pass" + password = "valid_password_123" + requests.post(f"{BASE_URL}/api/v1/users", json={ + "username": username, "display_name": "Test", "password": password + }) + + resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions", auth=(username, "wrong_password")) + assert resp.status_code == 401 + + +def test_logout(): + token = register_and_login("test_logout", "valid_password_123") + headers = {"Authorization": f"Bearer {token}"} + assert requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).status_code == 200 + assert requests.delete(f"{BASE_URL}/api/v1/auth/sessions/current", headers=headers).status_code == 204 + assert requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).status_code == 401 + + +def test_expiry(): + token = register_and_login("test_expiry", "valid_password_123") + headers = {"Authorization": f"Bearer {token}"} + + assert requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).status_code == 200 + + time.sleep(3) + + assert requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).status_code == 401 \ No newline at end of file