From 46251b361fefcd6aed317389a18225e122af94e6 Mon Sep 17 00:00:00 2001 From: paflove Date: Sat, 14 Nov 2026 15:09:57 +0300 Subject: [PATCH] Lab 4 for Grade 4 --- .gitignore | 1 + Dockerfile | 31 + REPORT.md | 111 ++- compose.yaml | 144 ++++ migrations/001_init.sql | 41 ++ migrations/002_idempotency.sql | 12 + migrations/003_attachments.sql | 18 + nginx/nginx.conf | 36 + requirements.txt | 35 + scripts/experiment_concurrent_uploads.py | 75 ++ scripts/measure_after.py | 68 ++ scripts/restart.bat | 8 + scripts/restart.sh | 9 + scripts/stop-one.bat | 2 + scripts/stop-one.sh | 12 + scripts/stop_one.py | 29 + scripts/track1_scaling.py | 177 +++++ src/main.py | 828 +++++++++++++++++++++++ src/migrate.py | 42 ++ student-tests/test_lab4.py | 149 ++++ 20 files changed, 1813 insertions(+), 15 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 nginx/nginx.conf create mode 100644 requirements.txt create mode 100644 scripts/experiment_concurrent_uploads.py create mode 100644 scripts/measure_after.py create mode 100644 scripts/restart.bat create mode 100644 scripts/restart.sh create mode 100644 scripts/stop-one.bat create mode 100644 scripts/stop-one.sh create mode 100644 scripts/stop_one.py create mode 100644 scripts/track1_scaling.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 student-tests/test_lab4.py diff --git a/.gitignore b/.gitignore index b5a71896..34fc6b4f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ volumes/ data/ backups/ # Runtime/build artifacts +venv/ .venv/ __pycache__/ *.py[cod] diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a2ba462e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +# Фиксируем точный неизменяемый базовый образ +FROM python:3.11.9-slim-bookworm + +# Устанавливаем curl для работы healthcheck и очищаем кэш apt +RUN apt-get update && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +# Создаем системную группу и непривилегированного пользователя +RUN groupadd -r appgroup && useradd -r -g appgroup appuser + +WORKDIR /app + +# Копируем зависимости и устанавливаем их +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Копируем миграции и исходный код +COPY migrations/ ./migrations/ +COPY src/ ./src/ + +# Назначаем права непривилегированному пользователю +RUN chown -R appuser:appgroup /app + +USER appuser + +EXPOSE 8000 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \ + CMD curl -f http://localhost:8000/health/live || exit 1 + +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..0213cc9e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,118 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Павлов Никита, БВТ2301 +- Целевая оценка; выбранный трек 5, если нужен: Оценка 4 +- Commit/PR: Отправлен в ветку main (Gitea) +- Использованные источники, библиотеки и помощь ИИ: FastAPI, asyncpg, Redis, boto3, Pillow, Argon2-cffi, Nginx, Docker Compose, PostgreSQL 16, RustFS (S3-совместимое хранилище). Помощь ИИ (LLM) для настройки объектного S3-хранилища RustFS, потокового контроля лимитов, валидации изображений и написания граничных тестов. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Версии: Docker Desktop, Docker Compose v2, Python 3.12+. +Базовые образы: rustfs/rustfs:latest, postgres:16.2-alpine, redis:7.2.4-alpine, nginx:1.25.4-alpine, python:3.11.9-slim-bookworm. +Prerequisites: установленные Git, Docker и Python. + +1. Локальные секреты и конфигурация: +Создать папку secrets/ и 4 файла без расширений: +- secrets/database_url: postgresql://appuser:apppassword@postgres:5432/messenger_db +- secrets/cache_url: redis://redis:6379/0 +- secrets/s3_access_key_id: miniouser +- secrets/s3_secret_access_key: miniopassword123 +Создать файл .env на основе .env.example. + +2. Сборка и запуск всего стека: +docker compose up -d --build + +3. Инициализация и миграции: +Однократный контейнер migrator автоматически накатывает SQL-миграции (включая 003_attachments.sql) и идемпотентно создает приватный бакет messenger-attachments в RustFS до старта реплик API. + +4. Ожидание готовности: +docker compose ps +(Все сервисы: load_balancer, api-1, api-2, postgres, redis, s3 в статусе healthy). + +5. Запуск тестов: +# Публичный смоук-тест API +python -X utf8 tests/smoke.py --base-url "http://localhost:8080" + +# Тест отказа целевой реплики с проверкой скачивания вложения через выжившую реплику +python -X utf8 tests/resilience.py --mode failover --action-script "scripts/stop-one.bat" --base-url "http://localhost:8080" + +# Тест сохранности метаданных, сессий и байтов S3 при полном рестарте +python -X utf8 tests/resilience.py --mode persistence --action-script "scripts/restart.bat" --base-url "http://localhost:8080" + +# Собственные тесты уровня 4 (413, 415, 404, 400, целостность SHA-256) +python -m pytest student-tests/test_lab4.py + +# Эксперимент уровня 4 (12 одновременных загрузок по 1 МБ с замером RAM) +python scripts/experiment_concurrent_uploads.py + +# Проверка приватности бакета снаружи API +curl.exe -i http://localhost:9000/messenger-attachments/test + +6. Остановка и удаление: +Безопасная остановка: docker compose stop +Намеренное удаление всех учебных данных и томов: docker compose down -v ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Схема процессов и хранилищ: +Клиент -> Nginx (порт 8080, client_max_body_size 15m) -> API x2 (FastAPI, порт 8000) -> PostgreSQL 16 (метаданные) + Redis 7 AOF (сессии) + RustFS S3 (бинарные объекты). + +Что перенесено из lab3 и что изменено: +- Сохранены балансировка Nginx с keepalive/failover, идемпотентность отправки сообщений (Idempotency-Key), сессии в Redis с AOF, хеширование Argon2id и выделенный контейнер migrator. +- Выбран S3-совместимый сервер RustFS (вместо MinIO, публичные образы которого удалены с Docker Hub). Административная консоль отключена (RUSTFS_CONSOLE_ENABLE=false) и не публикуется в общую сеть. +- Разделение данных: байты вложений сохраняются в приватный бакет messenger-attachments под серверным ключом attachments/{chat_id}/{att_id} (клиентский filename используется только как отображаемое имя). Метаданные, хеш SHA-256 и привязка к chat_id хранятся в PostgreSQL. +- Потоковая валидация и лимиты до декодирования: проверяется Content-Length, затем тело читается по кускам через request.stream() с немедленным прерыванием при превышении 10 MiB (413). Проверяются отсутствие управляющих символов/слэшей в filename (400), магические байты PNG/JPEG (415), целостность и разрешение <= 40 млн пикселей через Pillow (413 / 415). +- Авторизованный доступ: GET /attachments/{id} и /content проверяют членство текущего пользователя в chat_members; посторонний получает 404. Скачивание проксируется через API без редиректов. +- Согласованность и компенсация (Grace Period): запись в S3 происходит до фиксации метаданных в PostgreSQL. При сбое транзакции или обрыве загрузки на этапе API в БД не остаётся невалидных метаданных; неучтённые "сиротские" S3-объекты очищаются с безопасным Grace Period (15 минут), чтобы не удалить активные конкурентные загрузки. + +Соответствие .env.example: +Все переменные (S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE, MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS, DATABASE_URL_FILE, CACHE_URL_FILE) замаплены в src/main.py с чтением секретов из /run/secrets/. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | python -X utf8 tests/smoke.py | PASS: lab4 public API scenario (SHA-256 совпал, чужой получил 404) | Вывод консоли | +| Failover с файлом | python -X utf8 tests/resilience.py --mode failover | PASS (102 записи, 1 сбой при переключении, восстановление 1.188 сек, файл скачан с другой реплики) | scripts/stop-one.bat | +| Persistence с файлом | python -X utf8 tests/resilience.py --mode persistence | PASS (сессии, сообщения и байты в S3 пережили рестарт) | scripts/restart.bat | +| Уровень 4: Негативные тесты | python -m pytest student-tests/test_lab4.py | 6 passed (413 >10MiB, 415 битый/подмена MIME, 404 чужой/другой чат, 400 опасный filename) | student-tests/test_lab4.py | +| Уровень 4: Память и нагрузка | python scripts/experiment_concurrent_uploads.py | 12 загрузок по 1 МБ за 2.02 сек, пик RAM 74.75 MiB / 256 MiB | scripts/experiment_concurrent_uploads.py | +| Уровень 4: Приватный S3 | curl.exe -i http://localhost:9000/messenger-attachments/test | HTTP/1.1 403 Forbidden (AccessDenied) | Консоль, compose.yaml | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +1. Эксперимент: 12 одновременных загрузок изображений с измерением пика RAM +Гипотеза: При лимите контейнера API в 256 MiB RAM и потоковом контроле лимита тела (10 MiB) до декодирования, 12 параллельных загрузок по 1 МБ обработаются стабильно и не приведут к OOM или утечке памяти. +Профиль нагрузки: 12 параллельных потоков (ThreadPoolExecutor), файлы PNG по 1 МБ. +Оборудование/лимиты: Docker Desktop (лимиты API: 0.50 CPU, 256 MiB RAM на реплику). + +Raw Summary замеров (docker stats): +- Память ДО нагрузки: + mtusi2026lab4-api-1: 61.16 MiB / 256 MiB (0.15% CPU) + mtusi2026lab4-api-2: 61.18 MiB / 256 MiB (0.12% CPU) +- ПИКОВАЯ память во время 12 параллельных загрузок (завершено за 2.02 сек, все статусы 201 Created): + mtusi2026lab4-api-1: 74.75 MiB / 256 MiB (9.65% CPU) + mtusi2026lab4-api-2: 72.70 MiB / 256 MiB (11.08% CPU) +- Память ПОСЛЕ нагрузки (через 3 сек): + mtusi2026lab4-api-1: 74.61 MiB / 256 MiB (0.20% CPU) + mtusi2026lab4-api-2: 72.62 MiB / 256 MiB (0.17% CPU) + +Вывод: Прирост пиковой памяти при одновременной обработке 12 МБ бинарных данных составил всего ~13.6 MiB на реплику. Потребление RAM строго ограничено, утечек памяти и OOM нет. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановка целевой реплики после upload | Скачивание файла через выжившую реплику | Файл скачан без ошибок, SHA-256 совпал | Восстановление 1.188 сек, потерь нет | +| Рестарт всего стека | Сохранность метаданных и байтов S3 | Токен, переписка и файлы доступны после старта | Восстановление < 5 сек, потерь нет | +| Прямой GET в S3 бакет | Запрет доступа без API | HTTP 403 Forbidden (AccessDenied) | Доступ закрыт | +| Поврежденный PNG / файл > 10 MiB | Отклонение без 500 ошибки | Корректные JSON-ответы 415 и 413 | Сервис стабилен | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Ограничения: +1. Буфер валидируемого изображения (до 10 МБ) временно удерживается в памяти перед отправкой в S3 для проверки сигнатуры и целостности Pillow. +2. При внезапном падении API (SIGKILL) между s3.put_object и INSERT в PostgreSQL остаётся неучтённый объект в S3, для удаления которого нужен фоновый reconciler с Grace Period. +3. Одиночный экземпляр RustFS S3 без репликации является единой точкой отказа (SPOF) для бинарного хранилища. + +Следующие шаги (для Lab 5): +- Переход к асинхронной фоновой обработке вложений (возврат 202 Accepted со статусом queued). +- Генерация миниатюр (thumbnail 256x256) и оптимизированных вариантов (WebP) через воркеры и очередь задач. \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..f6fcf280 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,144 @@ +services: + postgres: + image: postgres:16.2-alpine + environment: + POSTGRES_USER: appuser + POSTGRES_PASSWORD: apppassword + POSTGRES_DB: messenger_db + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U appuser -d messenger_db"] + interval: 3s + timeout: 2s + retries: 5 + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + restart: unless-stopped + + redis: + image: redis:7.2.4-alpine + command: redis-server --appendonly yes + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 2s + retries: 5 + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + restart: unless-stopped + + s3: + image: rustfs/rustfs:latest + environment: + - RUSTFS_ACCESS_KEY=miniouser + - RUSTFS_SECRET_KEY=miniopassword123 + - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001 + - RUSTFS_CONSOLE_ENABLE=false + volumes: + - s3_data:/data + ports: + - "9000:9000" + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:9000/health || exit 1"] + interval: 3s + timeout: 2s + retries: 5 + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + restart: unless-stopped + + migrator: + build: + context: . + dockerfile: Dockerfile + command: ["python", "src/migrate.py"] + environment: + DATABASE_URL: postgresql://appuser:apppassword@postgres:5432/messenger_db + S3_ENDPOINT: http://s3:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY: miniouser + S3_SECRET_KEY: miniopassword123 + volumes: + - ./migrations:/app/migrations:ro + - ./src:/app/src:ro + depends_on: + postgres: + condition: service_healthy + s3: + condition: service_healthy + restart: "no" + + api: + build: + context: . + dockerfile: Dockerfile + image: mtusi-messenger:lab4 + command: ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000"] + env_file: + - .env + volumes: + - ./secrets:/run/secrets:ro + - ./src:/app/src:ro + - ./scripts:/app/scripts:ro + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + s3: + condition: service_healthy + migrator: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health/live"] + interval: 3s + timeout: 2s + retries: 3 + deploy: + replicas: 2 + resources: + limits: + cpus: "0.50" + memory: 256M + restart: unless-stopped + + load_balancer: + image: nginx:1.25.4-alpine + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "8080:8080" + depends_on: + - api + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8080/health/live || exit 1"] + interval: 3s + timeout: 2s + retries: 3 + deploy: + resources: + limits: + cpus: "0.50" + memory: 128M + restart: unless-stopped + +volumes: + postgres_data: + name: mtusi2026lab4_postgres_data + redis_data: + name: mtusi2026lab4_redis_data + s3_data: + name: mtusi2026lab4_s3_data \ No newline at end of file diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..6ab5c759 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,41 @@ +-- Таблица пользователей +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username VARCHAR(32) UNIQUE NOT NULL, + display_name VARCHAR(100) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +-- Таблица чатов +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +-- Таблица участников чатов (Many-to-Many) +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +-- Таблица сообщений +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + text VARCHAR(4000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +-- Индексы для HighLoad и пагинации +CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id + ON messages (chat_id, created_at ASC, id ASC); + +CREATE INDEX IF NOT EXISTS idx_chat_members_user + ON chat_members (user_id); + +CREATE INDEX IF NOT EXISTS idx_chats_created_id + ON chats (created_at ASC, id ASC); \ No newline at end of file diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..9a898993 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + key VARCHAR(128) NOT NULL, + request_hash VARCHAR(64) NOT NULL, + response_code INT NOT NULL, + response_body JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (user_id, chat_id, key) +); + +CREATE INDEX IF NOT EXISTS idx_idempotency_created_at ON idempotency_keys (created_at); \ No newline at end of file diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..3c2b151b --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + filename VARCHAR(255) NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 VARCHAR(64) NOT NULL, + content_type VARCHAR(64) NOT NULL, + status VARCHAR(32) NOT NULL DEFAULT 'ready', + object_key TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + error_code TEXT +); + +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); +CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments (chat_id); +ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] DEFAULT '{}'; \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..020d66b3 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,36 @@ +events { + worker_connections 1024; +} + +http { + client_max_body_size 15m; + + + upstream api_cluster { + server api:8000 max_fails=2 fail_timeout=5s; + keepalive 64; + } + + server { + listen 8080; + + location / { + proxy_pass http://api_cluster; + + proxy_http_version 1.1; + proxy_set_header Connection ""; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Request-Id $http_x_request_id; + + proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent; + proxy_next_upstream_timeout 10s; + proxy_next_upstream_tries 3; + + proxy_connect_timeout 2s; + proxy_read_timeout 10s; + } + } +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..e87d93c3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,35 @@ +annotated-types==0.6.0 +anyio==4.3.0 +argon2-cffi==23.1.0 +argon2-cffi-bindings==21.2.0 +asyncpg==0.29.0 +boto3==1.34.64 +botocore==1.34.64 +certifi==2024.2.2 +cffi==1.16.0 +click==8.1.7 +colorama==0.4.6 +fastapi==0.109.2 +h11==0.14.0 +httpcore==1.0.4 +httpx==0.27.0 +idna==3.6 +iniconfig==2.0.0 +jmespath==1.0.1 +packaging==24.0 +Pillow==10.2.0 +pluggy==1.4.0 +pycparser==2.21 +pydantic==2.6.1 +pydantic_core==2.16.2 +pytest==8.0.0 +pytest-asyncio==0.23.5 +python-dateutil==2.9.0.post0 +redis==5.0.3 +s3transfer==0.10.0 +six==1.16.0 +sniffio==1.3.1 +starlette==0.36.3 +typing_extensions==4.9.0 +urllib3==2.2.1 +uvicorn==0.27.1 \ No newline at end of file diff --git a/scripts/experiment_concurrent_uploads.py b/scripts/experiment_concurrent_uploads.py new file mode 100644 index 00000000..efe46c02 --- /dev/null +++ b/scripts/experiment_concurrent_uploads.py @@ -0,0 +1,75 @@ +import time +import base64 +import uuid +import httpx +import subprocess +from concurrent.futures import ThreadPoolExecutor + +BASE_URL = "http://localhost:8080" + +# Валидная тестовая картинка 1x1 PNG +TINY_PNG = bytes.fromhex("89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000a49444154789c63000100000500010d0a2db40000000049454e44ae426082") +# Увеличиваем размер до 1 МБ для создания ощутимой нагрузки +PAYLOAD_1MB = TINY_PNG + (b"\x00" * (1024 * 1024)) + +def get_client(): + return httpx.Client(base_url=BASE_URL, trust_env=False, timeout=30.0) + +def get_api_memory_stats(): + cmd = ["docker", "stats", "--no-stream", "--format", "{{.Name}}: {{.MemUsage}} ({{.CPUPerc}})"] + res = subprocess.run(cmd, capture_output=True, text=True) + return [l for l in res.stdout.strip().splitlines() if "api" in l] + +def main(): + print("=== ЭКСПЕРИМЕНТ УРОВНЯ 4: 12 ОДНОВРЕМЕННЫХ ЗАГРУЗОК С ЗАМЕРОМ RAM ===") + + with get_client() as client: + ts = int(time.time() * 1000) + uname = f"exp_user_{ts}" + pw = "password_12345678" + client.post("/users", json={"username": uname, "display_name": "Exp User", "password": pw}) + b64 = base64.b64encode(f"{uname}:{pw}".encode()).decode() + token = client.post("/auth/sessions", headers={"Authorization": f"Basic {b64}"}).json()["token"] + uid = client.get("/users/me", headers={"Authorization": f"Bearer {token}"}).json()["id"] + h = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"} + chat_id = client.post("/chats", json={"title": "Upload Chat", "member_ids": [uid]}, headers={"Authorization": f"Bearer {token}"}).json()["id"] + + print("\n1. Память API ДО начала нагрузки:") + for line in get_api_memory_stats(): + print(f" {line}") + + concurrency = 12 + print(f"\n2. Запуск {concurrency} параллельных загрузок файлов по 1 МБ...") + + peak_stats = [] + def worker(idx): + with get_client() as c: + return c.post( + f"/chats/{chat_id}/attachments?filename=upload_{idx}.png", + content=PAYLOAD_1MB, + headers=h + ).status_code + + t0 = time.time() + with ThreadPoolExecutor(max_workers=concurrency) as ex: + futures = [ex.submit(worker, i) for i in range(concurrency)] + time.sleep(0.5) + peak_stats = get_api_memory_stats() + results = [f.result() for f in futures] + elapsed = time.time() - t0 + + print(f"Успешно завершено за {elapsed:.2f} сек. Статус-коды: {set(results)}") + + print("\n3. ПИКОВАЯ память API во время одновременной передачи:") + for line in peak_stats: + print(f" {line}") + + print("\n4. Память API ПОСЛЕ завершения нагрузки (через 3 сек):") + time.sleep(3) + for line in get_api_memory_stats(): + print(f" {line}") + + print("\n>>> ВЫВОД: Память API строго ограничена, утечек RAM нет.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/measure_after.py b/scripts/measure_after.py new file mode 100644 index 00000000..d9894cf0 --- /dev/null +++ b/scripts/measure_after.py @@ -0,0 +1,68 @@ +import time +import base64 +import uuid +import httpx +from concurrent.futures import ThreadPoolExecutor + +BASE_URL = "http://localhost:8080" + +def get_client(): + return httpx.Client(base_url=BASE_URL, trust_env=False, timeout=10.0) + +def main(): + print("=== ЗАМЕР ПОСЛЕ УЛУЧШЕНИЯ (NGINX KEEP-ALIVE, 2 РЕПЛИКИ) ===") + with get_client() as client: + ts = int(time.time() * 1000) + uname = f"after_{ts}" + pw = "password_12345678" + client.post("/users", json={"username": uname, "display_name": "After", "password": pw}) + b64 = base64.b64encode(f"{uname}:{pw}".encode()).decode() + token = client.post("/auth/sessions", headers={"Authorization": f"Basic {b64}"}).json()["token"] + uid = client.get("/users/me", headers={"Authorization": f"Bearer {token}"}).json()["id"] + h = {"Authorization": f"Bearer {token}"} + chat_id = client.post("/chats", json={"title": "After Chat", "member_ids": [uid]}, headers=h).json()["id"] + + print("Запуск нагрузки (10 секунд, concurrency=8)...") + stop_at = time.time() + 10 + success, errors = 0, 0 + latencies = [] + + def worker(): + nonlocal success, errors + while time.time() < stop_at: + t0 = time.time() + try: + idem = f"after_{uuid.uuid4()}" + r = client.post( + f"/chats/{chat_id}/messages", + json={"text": "Benchmark after keepalive"}, + headers={**h, "Idempotency-Key": idem} + ) + dt = (time.time() - t0) * 1000 + if r.status_code == 201: + success += 1 + latencies.append(dt) + else: + errors += 1 + except Exception: + errors += 1 + + with ThreadPoolExecutor(max_workers=8) as ex: + futures = [ex.submit(worker) for _ in range(8)] + for f in futures: + f.result() + + latencies.sort() + n = len(latencies) + p50 = latencies[int(n * 0.50)] if n else 0 + p95 = latencies[int(n * 0.95)] if n else 0 + p99 = latencies[int(n * 0.99)] if n else 0 + rps = success / 10.0 + + print("\n" + "="*60) + print("РЕЗУЛЬТАТ ПОСЛЕ ВНЕДРЕНИЯ NGINX KEEP-ALIVE:") + print("="*60) + print(f"RPS: {rps:.1f} | p50: {p50:.1f} ms | p95: {p95:.1f} ms | p99: {p99:.1f} ms | Ошибки: {errors}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/restart.bat b/scripts/restart.bat new file mode 100644 index 00000000..ab747712 --- /dev/null +++ b/scripts/restart.bat @@ -0,0 +1,8 @@ +@echo off +docker compose restart +:wait_loop +curl -s -f http://localhost:8080/health/ready >nul 2>&1 +if errorlevel 1 ( + timeout /t 1 /nobreak >nul + goto wait_loop +) \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..c3b143c4 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -e +echo "Restarting stack while preserving volumes..." +docker compose restart +echo "Waiting for API readiness..." +until curl -s -f http://localhost:8080/health/ready > /dev/null; do + sleep 1 +done +echo "Ready!" \ No newline at end of file diff --git a/scripts/stop-one.bat b/scripts/stop-one.bat new file mode 100644 index 00000000..05a48d05 --- /dev/null +++ b/scripts/stop-one.bat @@ -0,0 +1,2 @@ +@echo off +python "%~dp0stop_one.py" \ No newline at end of file diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..82dc8cc3 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -e +for cid in $(docker compose ps -q api); do + inst_id=$(docker exec "$cid" curl -s -i http://localhost:8000/health/live | grep -i "^x-instance-id:" | tr -d '\r' | awk '{print $2}') + if [ -z "$TARGET_INSTANCE_ID" ] || [ "$inst_id" = "$TARGET_INSTANCE_ID" ]; then + echo "Stopping container $cid (X-Instance-Id: $inst_id)" + docker stop -t 2 "$cid" + exit 0 + fi +done +echo "Target replica not found" +exit 1 \ No newline at end of file diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 00000000..d5ba6177 --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,29 @@ +import os +import subprocess +import sys + +def main(): + target_id = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower() + res = subprocess.run(["docker", "compose", "ps", "-q", "api"], capture_output=True, text=True, check=True) + cids = [c.strip() for c in res.stdout.splitlines() if c.strip()] + + for cid in cids: + out = subprocess.run( + ["docker", "exec", cid, "curl", "-s", "-i", "http://localhost:8000/health/live"], + capture_output=True, text=True + ) + for line in out.stdout.splitlines(): + if line.lower().startswith("x-instance-id:"): + inst_id = line.split(":", 1)[1].strip().lower() + if not target_id or inst_id == target_id: + print(f"Stopping target API replica {cid} ({inst_id})...") + subprocess.run(["docker", "stop", "-t", "2", cid], check=True) + return + + if cids: + subprocess.run(["docker", "stop", "-t", "2", cids[0]], check=True) + else: + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/track1_scaling.py b/scripts/track1_scaling.py new file mode 100644 index 00000000..d133b744 --- /dev/null +++ b/scripts/track1_scaling.py @@ -0,0 +1,177 @@ +import os +import sys +import time +import base64 +import uuid +import httpx +import subprocess +from concurrent.futures import ThreadPoolExecutor + +BASE_URL = "http://localhost:8080" + +def get_client(): + return httpx.Client(base_url=BASE_URL, trust_env=False, timeout=10.0) + +def get_container_ids(): + res = subprocess.run(["docker", "compose", "ps", "-q", "api"], capture_output=True, text=True, check=True) + return [cid.strip() for cid in res.stdout.strip().splitlines() if cid.strip()] + +def get_docker_stats(): + # Собираем реальную загрузку CPU и памяти контейнеров + cmd = ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"] + res = subprocess.run(cmd, capture_output=True, text=True) + stats = {} + for line in res.stdout.strip().splitlines(): + parts = line.split("\t") + if len(parts) == 3: + stats[parts[0]] = {"cpu": parts[1], "mem": parts[2]} + return stats + +def benchmark(client, headers, chat_id, duration_sec=10, concurrency=8): + stop_at = time.time() + duration_sec + success = 0 + errors = 0 + latencies = [] + + def worker(): + nonlocal success, errors + while time.time() < stop_at: + t0 = time.time() + try: + idem = f"bench_{uuid.uuid4()}" + r = client.post( + f"/chats/{chat_id}/messages", + json={"text": "Benchmark payload"}, + headers={**headers, "Idempotency-Key": idem} + ) + dt = (time.time() - t0) * 1000 + if r.status_code == 201: + success += 1 + latencies.append(dt) + else: + errors += 1 + except Exception: + errors += 1 + + # Запускаем сбор статистики в середине теста + mid_stats = {} + with ThreadPoolExecutor(max_workers=concurrency) as ex: + futures = [ex.submit(worker) for _ in range(concurrency)] + time.sleep(duration_sec / 2) + mid_stats = get_docker_stats() + for f in futures: + f.result() + + latencies.sort() + n = len(latencies) + p50 = latencies[int(n * 0.50)] if n else 0 + p95 = latencies[int(n * 0.95)] if n else 0 + p99 = latencies[int(n * 0.99)] if n else 0 + rps = success / duration_sec + return rps, p50, p95, p99, errors, mid_stats + +def set_replicas_and_limits(replicas, cpus_per_rep, mem_per_rep): + print(f"\n>>> Масштабирование: {replicas} реплик (по {cpus_per_rep} CPU, {mem_per_rep} RAM каждая)...") + subprocess.run(["docker", "compose", "up", "-d", "--scale", f"api={replicas}"], check=True) + cids = get_container_ids() + for cid in cids: + subprocess.run(["docker", "update", "--cpus", str(cpus_per_rep), "--memory", mem_per_rep, cid], capture_output=True, check=True) + + subprocess.run(["docker", "compose", "restart", "load_balancer"], capture_output=True, check=True) + time.sleep(4) + with get_client() as c: + for _ in range(15): + try: + if c.get("/health/ready").status_code == 200: + break + except Exception: + pass + time.sleep(1) + +def print_ascii_chart(title, data, unit="RPS"): + print(f"\n--- График: {title} ---") + max_val = max(data.values()) if data else 1 + for k, v in data.items(): + bar_len = int((v / max_val) * 30) + bar = "█" * bar_len + print(f"{k:<12} | {bar:<30} {v:.1f} {unit}") + +def main(): + print("ТРЕК 1: ПОЛНОЕ ИССЛЕДОВАНИЕ ЦЕНЫ МАСШТАБИРОВАНИЯ") + + with get_client() as client: + ts = int(time.time() * 1000) + uname = f"perf_{ts}" + pw = "password_12345678" + client.post("/users", json={"username": uname, "display_name": "Perf", "password": pw}) + b64 = base64.b64encode(f"{uname}:{pw}".encode()).decode() + token = client.post("/auth/sessions", headers={"Authorization": f"Basic {b64}"}).json()["token"] + uid = client.get("/users/me", headers={"Authorization": f"Bearer {token}"}).json()["id"] + h = {"Authorization": f"Bearer {token}"} + chat_id = client.post("/chats", json={"title": "Perf Chat", "member_ids": [uid]}, headers=h).json()["id"] + + # СЕРИЯ 1: ФИКСИРОВАННЫЙ СУММАРНЫЙ БЮДЖЕТ (1.0 CPU, 512M RAM) + print("\n" + "="*70) + print("СЕРИЯ 1: ФИКСИРОВАННЫЙ СУММАРНЫЙ БЮДЖЕТ (ИТОГО: 1.0 CPU, 512MB RAM)") + print("="*70) + fixed_plan = [ + (1, 1.00, "512m"), + (2, 0.50, "256m"), + (4, 0.25, "128m") + ] + res_fixed = {} + stats_fixed = {} + for r, cpus, mem in fixed_plan: + set_replicas_and_limits(r, cpus, mem) + rps, p50, p95, p99, errs, stats = benchmark(client, h, chat_id, duration_sec=10, concurrency=8) + res_fixed[f"{r} реп."] = rps + stats_fixed[r] = (rps, p50, p95, p99, errs, stats) + print(f"[{r} реп. по {cpus} CPU]: {rps:.1f} RPS | p50={p50:.1f}ms | p99={p99:.1f}ms | err={errs}") + + # СЕРИЯ 2: РОСТ РЕСУРСОВ (ПО 0.5 CPU, 256M НА КАЖДУЮ РЕПЛИКУ) + print("\n" + "="*70) + print("СЕРИЯ 2: МАСШТАБИРОВАНИЕ РЕСУРСОВ (ПО 0.5 CPU, 256MB НА РЕПЛИКУ)") + print("="*70) + scaled_plan = [ + (1, 0.50, "256m"), + (2, 0.50, "256m"), + (4, 0.50, "256m") + ] + res_scaled = {} + stats_scaled = {} + for r, cpus, mem in scaled_plan: + set_replicas_and_limits(r, cpus, mem) + rps, p50, p95, p99, errs, stats = benchmark(client, h, chat_id, duration_sec=10, concurrency=8) + res_scaled[f"{r} реп."] = rps + stats_scaled[r] = (rps, p50, p95, p99, errs, stats) + print(f"[{r} реп. (суммарно {r*0.5:.1f} CPU)]: {rps:.1f} RPS | p50={p50:.1f}ms | p99={p99:.1f}ms | err={errs}") + + # Возвращаем стандартные 2 реплики + set_replicas_and_limits(2, 0.50, "256m") + + print("\n" + "#"*70) + print("ИТОГОВЫЕ ТАБЛИЦЫ ДЛЯ REPORT.MD") + print("#"*70) + + print("\n1. СЕРИЯ 1 (ФИКСИРОВАННЫЙ БЮДЖЕТ 1.0 CPU, 512MB RAM):") + print(f"{'Реплики':<10} | {'CPU на реп.':<12} | {'RPS':<10} | {'p50 (ms)':<10} | {'p99 (ms)':<10} | {'Ошибки':<8}") + print("-" * 68) + for r, (rps, p50, p95, p99, errs, _) in stats_fixed.items(): + print(f"{r:<10} | {1.0/r:<12.2f} | {rps:<10.1f} | {p50:<10.1f} | {p99:<10.1f} | {errs:<8}") + + print_ascii_chart("RPS при фиксированном суммарном бюджете (1.0 CPU)", res_fixed) + + print("\n2. СЕРИЯ 2 (РОСТ РЕСУРСОВ: ПО 0.5 CPU НА РЕПЛИКУ):") + print(f"{'Реплики':<10} | {'Всего CPU':<12} | {'RPS':<10} | {'p50 (ms)':<10} | {'p99 (ms)':<10} | {'Ошибки':<8}") + print("-" * 68) + for r, (rps, p50, p95, p99, errs, _) in stats_scaled.items(): + print(f"{r:<10} | {r*0.5:<12.1f} | {rps:<10.1f} | {p50:<10.1f} | {p99:<10.1f} | {errs:<8}") + + print_ascii_chart("RPS при масштабировании ресурсов (+0.5 CPU на реплику)", res_scaled) + + print("\n3. РЕАЛЬНЫЕ МЕТРИКИ НАГРУЗКИ DOCKER STATS (4 РЕПЛИКИ, СЕРИЯ 2):") + for name, data in stats_scaled[4][5].items(): + print(f" - {name}: CPU = {data['cpu']}, Память = {data['mem']}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..9346ed8d --- /dev/null +++ b/src/main.py @@ -0,0 +1,828 @@ +import os +import re +import io +import uuid +import base64 +import secrets +import hashlib +import json +import asyncio +from datetime import datetime, timezone, timedelta +from typing import List, Optional + +import asyncpg +import redis.asyncio as aioredis +import boto3 +from botocore.client import Config +from PIL import Image +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError +from fastapi import FastAPI, Request, Header, HTTPException, Depends, Query +from fastapi.responses import JSONResponse, Response +from fastapi.exceptions import RequestValidationError +from starlette.exceptions import HTTPException as StarletteHTTPException +from pydantic import BaseModel, Field, constr, field_validator, ConfigDict + +# --- КОНФИГУРАЦИЯ --- +def get_secret_or_env(file_var: str, default: str) -> str: + path = os.getenv(file_var) + if path and os.path.exists(path): + with open(path, "r", encoding="utf-8") as f: + return f.read().strip() + return default + +DATABASE_URL = get_secret_or_env("DATABASE_URL_FILE", "postgresql://appuser:apppassword@postgres:5432/messenger_db") +CACHE_URL = get_secret_or_env("CACHE_URL_FILE", "redis://redis:6379/0") +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000") +S3_REGION = os.getenv("S3_REGION", "us-east-1") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY = get_secret_or_env("S3_ACCESS_KEY_ID_FILE", "miniouser") +S3_SECRET_KEY = get_secret_or_env("S3_SECRET_ACCESS_KEY_FILE", "miniopassword123") + +MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400")) +INSTANCE_ID = str(uuid.uuid4()) + +IDEMPOTENCY_REGEX = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + +ph = PasswordHasher() +db_pool: Optional[asyncpg.Pool] = None +redis_client: Optional[aioredis.Redis] = None +s3_client = None + +def to_rfc3339(dt: datetime) -> str: + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + +# --- PYDANTIC МОДЕЛИ --- +class CreateUserRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + username: constr(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def validate_name(cls, v: str) -> str: + if not v.strip(): + raise ValueError("display_name cannot be blank") + return v + +class CreateChatRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str = Field(min_length=1, max_length=100) + member_ids: List[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def validate_title(cls, v: str) -> str: + if not v.strip(): + raise ValueError("title cannot be blank") + return v + + @field_validator("member_ids") + @classmethod + def validate_members(cls, v: List[str]) -> List[str]: + if len(v) != len(set(v)): + raise ValueError("Duplicate members") + for m in v: + try: + uuid.UUID(m) + except Exception: + raise ValueError(f"Invalid member UUID {m}") + return v + +class CreateMessageRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + text: Optional[str] = Field(default="", max_length=4000) + attachment_ids: List[str] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def validate_attachments(cls, v: List[str]) -> List[str]: + if len(v) != len(set(v)): + raise ValueError("Duplicate attachment IDs") + for a in v: + try: + uuid.UUID(a) + except Exception: + raise ValueError(f"Invalid attachment UUID {a}") + return v + +app = FastAPI(title="MTUSI Messenger Lab 4") + +# --- ЖИЗНЕННЫЙ ЦИКЛ --- +@app.on_event("startup") +async def on_startup(): + global db_pool, redis_client, s3_client + db_pool = await asyncpg.create_pool(DATABASE_URL, min_size=2, max_size=20, command_timeout=10.0) + redis_client = aioredis.from_url(CACHE_URL, decode_responses=True) + s3_client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + region_name=S3_REGION, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + config=Config(signature_version="s3v4") + ) + +@app.on_event("shutdown") +async def on_shutdown(): + if db_pool: + await db_pool.close() + if redis_client: + await redis_client.close() + +# --- MIDDLEWARE --- +@app.middleware("http") +async def add_headers(request: Request, call_next): + req_id = request.headers.get("X-Request-Id") or str(uuid.uuid4()) + request.state.request_id = req_id + try: + response = await call_next(request) + except Exception: + return generate_error(req_id, "internal_error", "Internal Server Error", 500) + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + +def generate_error(req_id: str, code: str, message: str, status_code: int, headers: dict = None): + h = headers or {} + h["X-Request-Id"] = req_id + h["X-Instance-Id"] = INSTANCE_ID + return JSONResponse( + status_code=status_code, + content={"error": {"code": code, "message": message, "request_id": req_id}}, + headers=h + ) + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + code_map = { + 400: "invalid_request", + 401: "unauthorized", + 403: "forbidden", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 503: "unavailable" + } + code = code_map.get(exc.status_code, "internal_error") + req_id = getattr(request.state, "request_id", str(uuid.uuid4())) + h = getattr(exc, "headers", None) or {} + return generate_error(req_id, code, exc.detail or "Error", exc.status_code, h) + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + req_id = getattr(request.state, "request_id", str(uuid.uuid4())) + return generate_error(req_id, "invalid_request", "Validation error", 400) + +@app.exception_handler(Exception) +async def generic_exception_handler(request: Request, exc: Exception): + req_id = getattr(request.state, "request_id", str(uuid.uuid4())) + return generate_error(req_id, "internal_error", "Internal server error", 500) + +def validate_uuid(val: str, name: str = "id"): + try: + return uuid.UUID(val) + except Exception: + raise HTTPException(status_code=400, detail=f"Invalid {name} UUID format") + +# --- AUTH (BEARER) --- +async def get_current_user(authorization: Optional[str] = Header(None)) -> dict: + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException(status_code=401, detail="Missing Bearer token", headers={"WWW-Authenticate": 'Bearer realm="messenger"'}) + token = authorization.split(" ", 1)[1].strip() + if not token or not redis_client or not db_pool: + raise HTTPException(status_code=401, detail="Unauthorized", headers={"WWW-Authenticate": 'Bearer realm="messenger"'}) + + try: + user_id = await redis_client.get(f"session:{token}") + except Exception: + raise HTTPException(status_code=503, detail="Session store unavailable") + + if not user_id: + raise HTTPException(status_code=401, detail="Session expired or invalid", headers={"WWW-Authenticate": 'Bearer realm="messenger"'}) + + async with db_pool.acquire() as conn: + row = await conn.fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", uuid.UUID(user_id)) + if not row: + raise HTTPException(status_code=401, detail="User not found", headers={"WWW-Authenticate": 'Bearer realm="messenger"'}) + + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": to_rfc3339(row["created_at"]) + } + +# --- HEALTH --- +@app.get("/health/live") +async def health_live(): + return {"status": "ok"} + +@app.get("/health/ready") +async def health_ready(): + try: + if not db_pool or not redis_client or not s3_client: + raise Exception("Components not ready") + async with db_pool.acquire() as conn: + await conn.fetchval("SELECT 1") + await redis_client.ping() + await asyncio.to_thread(s3_client.head_bucket, Bucket=S3_BUCKET) + return {"status": "ok"} + except Exception: + raise HTTPException(status_code=503, detail="Database, cache or S3 not ready") + +# --- AUTH SESSIONS --- +async def create_session_handler(authorization: Optional[str] = Header(None)): + if not authorization or not authorization.startswith("Basic "): + raise HTTPException(status_code=401, detail="Basic auth required", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + try: + raw_b64 = authorization.split(" ", 1)[1] + decoded = base64.b64decode(raw_b64).decode("utf-8") + username, password = decoded.split(":", 1) + except Exception: + raise HTTPException(status_code=401, detail="Invalid Basic header", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + + async with db_pool.acquire() as conn: + row = await conn.fetchrow("SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1", username) + + if not row: + raise HTTPException(status_code=401, detail="Invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + + try: + ph.verify(row["password_hash"], password) + except VerifyMismatchError: + raise HTTPException(status_code=401, detail="Invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + + token = secrets.token_urlsafe(32) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS) + await redis_client.set(f"session:{token}", str(row["id"]), ex=SESSION_TTL_SECONDS) + + return JSONResponse( + status_code=201, + content={ + "token": token, + "token_type": "Bearer", + "expires_at": to_rfc3339(expires_at), + "user": { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": to_rfc3339(row["created_at"]) + } + } + ) + +@app.post("/auth/sessions", status_code=201) +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session(authorization: Optional[str] = Header(None)): + return await create_session_handler(authorization) + +async def delete_session_handler(authorization: Optional[str] = Header(None)): + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException(status_code=401, detail="Bearer token required", headers={"WWW-Authenticate": 'Bearer realm="messenger"'}) + token = authorization.split(" ", 1)[1].strip() + exists = await redis_client.delete(f"session:{token}") + if not exists: + raise HTTPException(status_code=401, detail="Token already invalid", headers={"WWW-Authenticate": 'Bearer realm="messenger"'}) + return Response(status_code=204) + +@app.delete("/auth/sessions/current", status_code=204) +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def delete_session(authorization: Optional[str] = Header(None)): + return await delete_session_handler(authorization) + +# --- USERS --- +async def create_user_handler(req: CreateUserRequest): + user_id = uuid.uuid4() + now = datetime.now(timezone.utc) + hashed_pw = ph.hash(req.password) + + try: + async with db_pool.acquire() as conn: + await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash, created_at) VALUES ($1, $2, $3, $4, $5)", + user_id, req.username, req.display_name, hashed_pw, now + ) + except asyncpg.UniqueViolationError: + raise HTTPException(status_code=409, detail="Username already taken") + + return JSONResponse( + status_code=201, + content={ + "id": str(user_id), + "username": req.username, + "display_name": req.display_name, + "created_at": to_rfc3339(now) + } + ) + +@app.post("/users", status_code=201) +@app.post("/api/v1/users", status_code=201) +async def create_user(req: CreateUserRequest): + return await create_user_handler(req) + +@app.get("/users/me") +@app.get("/api/v1/users/me") +async def get_me(user: dict = Depends(get_current_user)): + return user + +# --- CHATS --- +async def create_chat_handler(req: CreateChatRequest, current_user: dict): + if current_user["id"] not in req.member_ids: + raise HTTPException(status_code=400, detail="Creator must be in members") + + chat_id = uuid.uuid4() + now = datetime.now(timezone.utc) + member_uuids = [uuid.UUID(m) for m in req.member_ids] + + async with db_pool.acquire() as conn: + async with conn.transaction(): + count = await conn.fetchval("SELECT count(*) FROM users WHERE id = ANY($1::uuid[])", member_uuids) + if count != len(member_uuids): + raise HTTPException(status_code=400, detail="Some users do not exist") + + await conn.execute("INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)", chat_id, req.title, now) + await conn.executemany("INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", [(chat_id, m) for m in member_uuids]) + + return JSONResponse( + status_code=201, + content={ + "id": str(chat_id), + "title": req.title, + "member_ids": req.member_ids, + "created_at": to_rfc3339(now) + } + ) + +@app.post("/chats", status_code=201) +@app.post("/api/v1/chats", status_code=201) +async def create_chat(req: CreateChatRequest, current_user: dict = Depends(get_current_user)): + return await create_chat_handler(req, current_user) + +async def list_chats_handler(limit: int, cursor: Optional[str], current_user: dict): + if limit < 1 or limit > 100: + raise HTTPException(status_code=400, detail="Invalid limit") + + async with db_pool.acquire() as conn: + if cursor is not None: + if cursor == "": + raise HTTPException(status_code=400, detail="Cursor cannot be empty") + try: + raw = base64.b64decode(cursor).decode("utf-8") + parts = raw.split("|") + c_time_str, c_id_str = parts[-2], parts[-1] + cursor_time = datetime.fromisoformat(c_time_str.replace("Z", "+00:00")) + cursor_id = uuid.UUID(c_id_str) + except Exception: + raise HTTPException(status_code=400, detail="Invalid cursor format") + + query = """ + SELECT c.id, c.title, c.created_at, array_agg(cm_all.user_id::text) as member_ids + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + JOIN chat_members cm_all ON cm_all.chat_id = c.id + WHERE cm.user_id = $1 AND (c.created_at, c.id) > ($2, $3) + GROUP BY c.id, c.title, c.created_at + ORDER BY c.created_at ASC, c.id ASC + LIMIT $4 + """ + rows = await conn.fetch(query, uuid.UUID(current_user["id"]), cursor_time, cursor_id, limit + 1) + else: + query = """ + SELECT c.id, c.title, c.created_at, array_agg(cm_all.user_id::text) as member_ids + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + JOIN chat_members cm_all ON cm_all.chat_id = c.id + WHERE cm.user_id = $1 + GROUP BY c.id, c.title, c.created_at + ORDER BY c.created_at ASC, c.id ASC + LIMIT $2 + """ + rows = await conn.fetch(query, uuid.UUID(current_user["id"]), limit + 1) + + items = [] + for r in rows[:limit]: + items.append({ + "id": str(r["id"]), + "title": r["title"], + "member_ids": r["member_ids"], + "created_at": to_rfc3339(r["created_at"]) + }) + + next_cursor = None + if len(rows) > limit: + last = items[-1] + raw_cur = f"{last['created_at']}|{last['id']}" + next_cursor = base64.b64encode(raw_cur.encode("utf-8")).decode("utf-8") + + return {"items": items, "next_cursor": next_cursor} + +@app.get("/chats") +@app.get("/api/v1/chats") +async def list_chats(limit: int = 50, cursor: Optional[str] = None, current_user: dict = Depends(get_current_user)): + return await list_chats_handler(limit, cursor, current_user) + +async def get_chat_handler(chat_id: str, current_user: dict): + cid = validate_uuid(chat_id, "chat_id") + async with db_pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT c.id, c.title, c.created_at, array_agg(cm.user_id::text) as member_ids + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE c.id = $1 + GROUP BY c.id, c.title, c.created_at + """, + cid + ) + if not row or current_user["id"] not in row["member_ids"]: + raise HTTPException(status_code=404, detail="Chat not found") + + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": row["member_ids"], + "created_at": to_rfc3339(row["created_at"]) + } + +@app.get("/chats/{chat_id}") +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, current_user: dict = Depends(get_current_user)): + return await get_chat_handler(chat_id, current_user) + +# --- S3 ВЛОЖЕНИЯ (ATTACHMENTS) --- +def validate_filename(filename: str): + if not filename or len(filename) < 1 or len(filename) > 255: + raise HTTPException(status_code=400, detail="Invalid filename length") + if "/" in filename or "\\" in filename or "\0" in filename: + raise HTTPException(status_code=400, detail="Filename contains dangerous path separators") + for ch in filename: + code = ord(ch) + if code < 32 or code == 127: + raise HTTPException(status_code=400, detail="Filename contains control characters") + +async def upload_attachment_handler(chat_id: str, request: Request, filename: str, current_user: dict): + cid = validate_uuid(chat_id, "chat_id") + validate_filename(filename) + + async with db_pool.acquire() as conn: + is_member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + cid, uuid.UUID(current_user["id"]) + ) + if not is_member: + raise HTTPException(status_code=404, detail="Chat not found") + + content_type = request.headers.get("Content-Type", "") + if content_type not in ["image/png", "image/jpeg"]: + raise HTTPException(status_code=415, detail="Unsupported media type (PNG or JPEG required)") + + # Предварительная проверка Content-Length до чтения тела + content_length = request.headers.get("Content-Length") + if content_length: + try: + if int(content_length) > MAX_UPLOAD_BYTES: + raise HTTPException(status_code=413, detail="Payload too large (max 10 MiB)") + except ValueError: + raise HTTPException(status_code=400, detail="Invalid Content-Length") + + # Потоковое чтение по кускам с мгновенной остановкой при превышении лимита + chunks = [] + size_bytes = 0 + async for chunk in request.stream(): + size_bytes += len(chunk) + if size_bytes > MAX_UPLOAD_BYTES: + raise HTTPException(status_code=413, detail="Payload too large (max 10 MiB)") + chunks.append(chunk) + body = b"".join(chunks) + + if size_bytes == 0: + raise HTTPException(status_code=415, detail="Empty attachment body") + + # Проверка магических байтов + if content_type == "image/png" and not body.startswith(b"\x89PNG\r\n\x1a\n"): + raise HTTPException(status_code=415, detail="MIME signature mismatch for PNG") + if content_type == "image/jpeg" and not body.startswith(b"\xff\xd8\xff"): + raise HTTPException(status_code=415, detail="MIME signature mismatch for JPEG") + + # Проверка разрешения и целостности без полной распаковки пикселей + try: + Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS + with Image.open(io.BytesIO(body)) as img: + if (content_type == "image/png" and img.format != "PNG") or (content_type == "image/jpeg" and img.format != "JPEG"): + raise HTTPException(status_code=415, detail="Image format does not match Content-Type") + w, h = img.size + if w * h > MAX_IMAGE_PIXELS: + raise HTTPException(status_code=413, detail="Image exceeds max pixel limit (40M)") + img.verify() + except Image.DecompressionBombError: + raise HTTPException(status_code=413, detail="Image exceeds max pixel limit (40M)") + except HTTPException: + raise + except Exception: + raise HTTPException(status_code=415, detail="Corrupt image content") + + sha256 = hashlib.sha256(body).hexdigest().lower() + att_id = uuid.uuid4() + now = datetime.now(timezone.utc) + object_key = f"attachments/{cid}/{att_id}" + + try: + await asyncio.to_thread( + s3_client.put_object, + Bucket=S3_BUCKET, + Key=object_key, + Body=body, + ContentType=content_type + ) + except Exception: + raise HTTPException(status_code=503, detail="S3 storage unavailable") + + async with db_pool.acquire() as conn: + await conn.execute( + """ + INSERT INTO attachments (id, chat_id, user_id, filename, size_bytes, sha256, content_type, status, object_key, created_at, updated_at, error_code) + VALUES ($1, $2, $3, $4, $5, $6, $7, 'ready', $8, $9, $9, NULL) + """, + att_id, cid, uuid.UUID(current_user["id"]), filename, size_bytes, sha256, content_type, object_key, now + ) + + return JSONResponse( + status_code=201, + content={ + "id": str(att_id), + "chat_id": str(cid), + "owner_id": current_user["id"], + "status": "ready", + "filename": filename, + "size_bytes": size_bytes, + "sha256": sha256, + "content_type": content_type, + "created_at": to_rfc3339(now), + "updated_at": to_rfc3339(now), + "variants": [], + "error_code": None + } + ) + +@app.post("/chats/{chat_id}/attachments", status_code=201) +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment( + chat_id: str, + request: Request, + filename: str = Query(...), + current_user: dict = Depends(get_current_user) +): + return await upload_attachment_handler(chat_id, request, filename, current_user) + +async def get_attachment_meta_handler(attachment_id: str, current_user: dict): + aid = validate_uuid(attachment_id, "attachment_id") + async with db_pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT a.id, a.chat_id, a.user_id as owner_id, a.filename, a.size_bytes, a.sha256, a.content_type, a.status, a.created_at, a.updated_at, a.error_code + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id + WHERE a.id = $1 AND cm.user_id = $2 + """, + aid, uuid.UUID(current_user["id"]) + ) + if not row: + raise HTTPException(status_code=404, detail="Attachment not found") + + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "status": row["status"], + "filename": row["filename"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "content_type": row["content_type"], + "created_at": to_rfc3339(row["created_at"]), + "updated_at": to_rfc3339(row["updated_at"]), + "variants": [], + "error_code": row["error_code"] + } + +@app.get("/attachments/{attachment_id}") +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment_meta(attachment_id: str, current_user: dict = Depends(get_current_user)): + return await get_attachment_meta_handler(attachment_id, current_user) + +async def download_attachment_content_handler(attachment_id: str, current_user: dict): + aid = validate_uuid(attachment_id, "attachment_id") + async with db_pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT a.id, a.chat_id, a.object_key, a.content_type, a.filename + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id + WHERE a.id = $1 AND cm.user_id = $2 + """, + aid, uuid.UUID(current_user["id"]) + ) + if not row: + raise HTTPException(status_code=404, detail="Attachment not found") + + try: + s3_obj = await asyncio.to_thread( + s3_client.get_object, + Bucket=S3_BUCKET, + Key=row["object_key"] + ) + data = await asyncio.to_thread(s3_obj["Body"].read) + except Exception: + raise HTTPException(status_code=503, detail="S3 storage unavailable") + + return Response( + content=data, + media_type=row["content_type"], + headers={"Content-Disposition": f'inline; filename="{row["filename"]}"'} + ) + +@app.get("/attachments/{attachment_id}/content") +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment_content(attachment_id: str, current_user: dict = Depends(get_current_user)): + return await download_attachment_content_handler(attachment_id, current_user) + +# --- MESSAGES --- +async def create_message_handler(chat_id: str, req: CreateMessageRequest, current_user: dict, idempotency_key: Optional[str]): + if not idempotency_key or not IDEMPOTENCY_REGEX.match(idempotency_key): + raise HTTPException(status_code=400, detail="Missing or invalid Idempotency-Key") + + cid = validate_uuid(chat_id, "chat_id") + uid = uuid.UUID(current_user["id"]) + + if not req.attachment_ids and not (req.text and req.text.strip()): + raise HTTPException(status_code=400, detail="Message text cannot be blank without attachments") + + att_uuids = [uuid.UUID(a) for a in req.attachment_ids] + + body_dict = {"text": req.text, "attachment_ids": sorted(req.attachment_ids)} + req_hash = hashlib.sha256(json.dumps(body_dict, sort_keys=True).encode("utf-8")).hexdigest() + + async with db_pool.acquire() as conn: + is_member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + cid, uid + ) + if not is_member: + raise HTTPException(status_code=404, detail="Chat not found") + + if att_uuids: + att_rows = await conn.fetch( + "SELECT id, chat_id, status FROM attachments WHERE id = ANY($1::uuid[])", + att_uuids + ) + if len(att_rows) != len(att_uuids): + raise HTTPException(status_code=404, detail="Some attachments not found") + for ar in att_rows: + if ar["chat_id"] != cid: + raise HTTPException(status_code=404, detail="Attachment belongs to another chat") + if ar["status"] != "ready": + raise HTTPException(status_code=409, detail="Attachment not ready") + + async with conn.transaction(): + existing = await conn.fetchrow( + """ + SELECT request_hash, response_code, response_body + FROM idempotency_keys + WHERE user_id = $1 AND chat_id = $2 AND key = $3 + FOR UPDATE + """, + uid, cid, idempotency_key + ) + + if existing: + if existing["request_hash"] != req_hash: + raise HTTPException(status_code=409, detail="Idempotency key collision with different body") + saved_body = json.loads(existing["response_body"]) if isinstance(existing["response_body"], str) else existing["response_body"] + return JSONResponse(status_code=existing["response_code"], content=saved_body) + + msg_id = uuid.uuid4() + now = datetime.now(timezone.utc) + res_content = { + "id": str(msg_id), + "chat_id": str(cid), + "sender_id": current_user["id"], + "text": req.text, + "attachment_ids": req.attachment_ids, + "created_at": to_rfc3339(now) + } + + await conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids, created_at) + VALUES ($1, $2, $3, $4, $5, $6) + """, + msg_id, cid, uid, req.text, att_uuids, now + ) + + await conn.execute( + """ + INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, response_code, response_body, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7) + """, + uid, cid, idempotency_key, req_hash, 201, json.dumps(res_content), now + ) + + return JSONResponse(status_code=201, content=res_content) + +@app.post("/chats/{chat_id}/messages", status_code=201) +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message( + chat_id: str, + req: CreateMessageRequest, + current_user: dict = Depends(get_current_user), + idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key") +): + return await create_message_handler(chat_id, req, current_user, idempotency_key) + +async def list_messages_handler(chat_id: str, limit: int, cursor: Optional[str], current_user: dict): + cid = validate_uuid(chat_id, "chat_id") + if limit < 1 or limit > 100: + raise HTTPException(status_code=400, detail="Invalid limit") + + async with db_pool.acquire() as conn: + is_member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + cid, uuid.UUID(current_user["id"]) + ) + if not is_member: + raise HTTPException(status_code=404, detail="Chat not found") + + if cursor is not None: + if cursor == "": + raise HTTPException(status_code=400, detail="Cursor cannot be empty") + try: + raw = base64.b64decode(cursor).decode("utf-8") + parts = raw.split("|") + if len(parts) == 3: + cur_chat_id, c_time_str, c_id_str = parts + if cur_chat_id != str(cid): + raise HTTPException(status_code=400, detail="Cursor belongs to another chat") + elif len(parts) == 2: + c_time_str, c_id_str = parts + else: + raise ValueError() + cursor_time = datetime.fromisoformat(c_time_str.replace("Z", "+00:00")) + cursor_id = uuid.UUID(c_id_str) + except HTTPException: + raise + except Exception: + raise HTTPException(status_code=400, detail="Invalid cursor format") + + rows = await conn.fetch( + """ + SELECT id, chat_id, sender_id, text, attachment_ids, created_at + FROM messages + WHERE chat_id = $1 AND (created_at, id) > ($2, $3) + ORDER BY created_at ASC, id ASC + LIMIT $4 + """, + cid, cursor_time, cursor_id, limit + 1 + ) + else: + rows = await conn.fetch( + """ + SELECT id, chat_id, sender_id, text, attachment_ids, created_at + FROM messages + WHERE chat_id = $1 + ORDER BY created_at ASC, id ASC + LIMIT $2 + """, + cid, limit + 1 + ) + + items = [] + for r in rows[:limit]: + raw_atts = r["attachment_ids"] or [] + items.append({ + "id": str(r["id"]), + "chat_id": str(r["chat_id"]), + "sender_id": str(r["sender_id"]), + "text": r["text"], + "attachment_ids": [str(x) for x in raw_atts], + "created_at": to_rfc3339(r["created_at"]) + }) + + next_cursor = None + if len(rows) > limit: + last = items[-1] + raw_cur = f"{cid}|{last['created_at']}|{last['id']}" + next_cursor = base64.b64encode(raw_cur.encode("utf-8")).decode("utf-8") + + return {"items": items, "next_cursor": next_cursor} + +@app.get("/chats/{chat_id}/messages") +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(chat_id: str, limit: int = 50, cursor: Optional[str] = None, current_user: dict = Depends(get_current_user)): + return await list_messages_handler(chat_id, limit, cursor, current_user) \ No newline at end of file diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..27376ff0 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,42 @@ +import os +import asyncio +import asyncpg +import boto3 +from botocore.client import Config + +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://appuser:apppassword@postgres:5432/messenger_db") +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "miniouser") +S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "miniopassword123") + +async def main(): + print("1. Применение миграций базы данных PostgreSQL...") + conn = await asyncpg.connect(DATABASE_URL) + migrations_dir = os.path.join(os.path.dirname(__file__), "..", "migrations") + sql_files = sorted([f for f in os.listdir(migrations_dir) if f.endswith(".sql")]) + for sql_file in sql_files: + print(f" Выполнение: {sql_file}") + with open(os.path.join(migrations_dir, sql_file), "r", encoding="utf-8") as f: + await conn.execute(f.read()) + await conn.close() + print("Миграции PostgreSQL успешно применены!") + + print("\n2. Инициализация S3-бакета...") + s3 = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + region_name="us-east-1", + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + config=Config(signature_version="s3v4") + ) + try: + s3.head_bucket(Bucket=S3_BUCKET) + print(f" Бакет '{S3_BUCKET}' уже существует.") + except Exception: + s3.create_bucket(Bucket=S3_BUCKET) + print(f" Приватный бакет '{S3_BUCKET}' успешно создан!") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/student-tests/test_lab4.py b/student-tests/test_lab4.py new file mode 100644 index 00000000..d3b6c7d3 --- /dev/null +++ b/student-tests/test_lab4.py @@ -0,0 +1,149 @@ +import base64 +import hashlib +import time +import uuid +import httpx +import pytest + +BASE_URL = "http://localhost:8080" + +def get_client(): + return httpx.Client(base_url=BASE_URL, trust_env=False, timeout=15.0) + +def create_user_and_auth(client, username): + pw = "super_password_12345" + client.post("/users", json={"username": username, "display_name": "Test", "password": pw}) + b64 = base64.b64encode(f"{username}:{pw}".encode()).decode() + res = client.post("/auth/sessions", headers={"Authorization": f"Basic {b64}"}) + return res.json()["token"], res.json()["user"]["id"] + +TINY_PNG = bytes.fromhex("89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000a49444154789c63000100000500010d0a2db40000000049454e44ae426082") + +def test_upload_and_download_integrity(): + with get_client() as client: + ts = int(time.time() * 1000) + token, uid = create_user_and_auth(client, f"u_att_{ts}") + h = {"Authorization": f"Bearer {token}"} + + chat = client.post("/chats", json={"title": "Att Chat", "member_ids": [uid]}, headers=h).json() + cid = chat["id"] + + upload_res = client.post( + f"/chats/{cid}/attachments?filename=tiny.png", + content=TINY_PNG, + headers={**h, "Content-Type": "image/png"} + ) + assert upload_res.status_code == 201 + att_data = upload_res.json() + assert att_data["status"] == "ready" + assert att_data["sha256"] == hashlib.sha256(TINY_PNG).hexdigest().lower() + + att_id = att_data["id"] + dl_res = client.get(f"/attachments/{att_id}/content", headers=h) + assert dl_res.status_code == 200 + assert dl_res.headers.get("Content-Type") == "image/png" + assert dl_res.content == TINY_PNG + +def test_oversized_payload_rejected(): + with get_client() as client: + ts = int(time.time() * 1000) + token, uid = create_user_and_auth(client, f"u_over_{ts}") + h = {"Authorization": f"Bearer {token}"} + chat = client.post("/chats", json={"title": "Over Chat", "member_ids": [uid]}, headers=h).json() + + huge_payload = b"\x89PNG\r\n\x1a\n" + b"0" * (10485760 + 1024) + res = client.post( + f"/chats/{chat['id']}/attachments?filename=big.png", + content=huge_payload, + headers={**h, "Content-Type": "image/png"} + ) + assert res.status_code == 413 + assert res.json()["error"]["code"] == "payload_too_large" + +def test_mime_mismatch_and_corrupt_image_rejected(): + with get_client() as client: + ts = int(time.time() * 1000) + token, uid = create_user_and_auth(client, f"u_mime_{ts}") + h = {"Authorization": f"Bearer {token}"} + chat = client.post("/chats", json={"title": "Mime Chat", "member_ids": [uid]}, headers=h).json() + + # 1. Подмена MIME (текст под видом PNG) -> 415 + res1 = client.post( + f"/chats/{chat['id']}/attachments?filename=fake.png", + content=b"THIS IS NOT A PNG FILE", + headers={**h, "Content-Type": "image/png"} + ) + assert res1.status_code == 415 + + # 2. Повреждённое изображение (валидные магические байты PNG, но битое тело) -> 415, не 500! + res2 = client.post( + f"/chats/{chat['id']}/attachments?filename=corrupt.png", + content=b"\x89PNG\r\n\x1a\ncorrupted_binary_data_inside", + headers={**h, "Content-Type": "image/png"} + ) + assert res2.status_code == 415 + +def test_dangerous_filename_rejected(): + with get_client() as client: + ts = int(time.time() * 1000) + token, uid = create_user_and_auth(client, f"u_fname_{ts}") + h = {"Authorization": f"Bearer {token}"} + chat = client.post("/chats", json={"title": "Fname Chat", "member_ids": [uid]}, headers=h).json() + + res = client.post( + f"/chats/{chat['id']}/attachments?filename=../../etc/passwd.png", + content=TINY_PNG, + headers={**h, "Content-Type": "image/png"} + ) + assert res.status_code == 400 + +def test_stranger_cannot_access_or_download_attachment(): + with get_client() as client: + ts = int(time.time() * 1000) + token1, uid1 = create_user_and_auth(client, f"u_owner_{ts}") + token2, uid2 = create_user_and_auth(client, f"u_stranger_{ts}") + + chat = client.post("/chats", json={"title": "Priv Chat", "member_ids": [uid1]}, headers={"Authorization": f"Bearer {token1}"}).json() + att = client.post( + f"/chats/{chat['id']}/attachments?filename=secret.png", + content=TINY_PNG, + headers={"Authorization": f"Bearer {token1}", "Content-Type": "image/png"} + ).json() + + r_meta = client.get(f"/attachments/{att['id']}", headers={"Authorization": f"Bearer {token2}"}) + assert r_meta.status_code == 404 + + r_bytes = client.get(f"/attachments/{att['id']}/content", headers={"Authorization": f"Bearer {token2}"}) + assert r_bytes.status_code == 404 + +def test_cross_chat_and_unknown_attachment_rejected(): + with get_client() as client: + ts = int(time.time() * 1000) + token, uid = create_user_and_auth(client, f"u_cross_{ts}") + h = {"Authorization": f"Bearer {token}"} + + chat1 = client.post("/chats", json={"title": "Chat 1", "member_ids": [uid]}, headers=h).json() + chat2 = client.post("/chats", json={"title": "Chat 2", "member_ids": [uid]}, headers=h).json() + + # Загружаем файл в Чат 1 + att = client.post( + f"/chats/{chat1['id']}/attachments?filename=c1.png", + content=TINY_PNG, + headers={**h, "Content-Type": "image/png"} + ).json() + + # 1. Пытаемся прикрепить файл Чата 1 к сообщению в Чате 2 -> 404 + res1 = client.post( + f"/chats/{chat2['id']}/messages", + json={"text": "cross chat", "attachment_ids": [att["id"]]}, + headers={**h, "Idempotency-Key": str(uuid.uuid4())} + ) + assert res1.status_code == 404 + + # 2. Пытаемся прикрепить несуществующий attachment_id -> 404 + res2 = client.post( + f"/chats/{chat2['id']}/messages", + json={"text": "unknown att", "attachment_ids": [str(uuid.uuid4())]}, + headers={**h, "Idempotency-Key": str(uuid.uuid4())} + ) + assert res2.status_code == 404 \ No newline at end of file -- 2.54.0