diff --git a/.dockerignore b/.dockerignore index bc16e556..036eee9e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,10 @@ __pycache__ node_modules evidence .DS_Store +.git +.env +.venv +__pycache__ +.pytest_cache +*.pyc +evidence \ No newline at end of file diff --git a/.env.example b/.env.example index d99d26a2..6182c3a7 100644 --- a/.env.example +++ b/.env.example @@ -18,3 +18,6 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +S3_ACCESS_KEY=change-me +S3_SECRET_KEY=change-me-too +S3_BUCKET=messenger-attachments \ No newline at end of file diff --git a/.gitignore b/.gitignore index b5a71896..702888b4 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,8 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +.env +.venv/ +__pycache__/ +*.pyc +evidence/*.tmp \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..66851c47 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.12-slim + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ ./src/ +COPY migrations/ ./migrations/ + +ENV HOST=0.0.0.0 +ENV PORT=8080 + +EXPOSE 8080 + +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..068e3c0f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,147 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- **Автор, группа:** Матвиенко, БПИ2304 +- **Целевая оценка:** 3 +- **Commit/PR:** добавить после публикации +- **Использованные источники, библиотеки и помощь ИИ:** материалы курса, OpenAPI-контракт и публичные тесты; Python 3.12, FastAPI, Uvicorn, psycopg, boto3, Pillow, PostgreSQL 16, RustFS 1.0.3, Nginx 1.29, Docker Compose. Помощь ИИ использовалась при разработке, отладке и оформлении отчёта. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Для запуска необходимы Docker Desktop с Docker Compose, Git и WSL Ubuntu с Python 3 и Make. + +В корне проекта создать `.env` с локальными значениями: + +```dotenv +POSTGRES_PASSWORD=<локальный_пароль> +SESSION_TTL_SECONDS=3600 +S3_ACCESS_KEY=<локальный_ключ> +S3_SECRET_KEY=<локальный_секрет> +S3_BUCKET=messenger-attachments +``` + +Секреты не должны попадать в Git. Адрес S3 внутри Docker-сети: `http://minio:9000` (сервис `minio` использует RustFS). + +Сборка и запуск: + +```bash +docker compose up -d --build +docker compose ps -a +curl http://127.0.0.1:8080/health/ready +``` + +Сервис `migrate` применяет SQL-миграции до запуска API, а `s3-init` создаёт приватный bucket. Повторная проверка: + +```bash +docker compose run --rm --no-deps migrate +docker compose run --rm s3-init +``` + +Публичные проверки: + +```bash +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +``` + +После проверок восстановить обе API-реплики: + +```bash +docker compose up -d --wait api1 api2 +``` + +Безопасная остановка с сохранением данных: + +```bash +docker compose down +``` + +Намеренное удаление учебных данных и постоянных томов: + +```bash +docker compose down -v +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +**Схема:** клиент → Nginx → API1 / API2 → PostgreSQL + RustFS (S3). + +Единая точка входа — `localhost:8080`. Nginx балансирует запросы между двумя независимыми экземплярами FastAPI. PostgreSQL является источником истины для пользователей, сессий, чатов, сообщений и метаданных вложений. Оригинальные файлы сохраняются в приватном S3-bucket RustFS, а не в PostgreSQL или локальных каталогах API. + +Из lab3 перенесены регистрация, аутентификация, чаты, сообщения, идемпотентность, балансировка и проверка отказа реплики. + +В lab4 добавлены: +- Загрузка оригинальных PNG/JPEG и авторизованное скачивание через API. +- Проверка MIME по содержимому, контроль размера до 10 MiB и количества пикселей до 40 миллионов. +- Проверка имени файла и создание серверного S3 object key. +- Таблицы `attachments` и `message_attachments`. +- Привязка изображений к сообщениям через `attachment_ids`. +- Проверка доступа к чату перед получением метаданных и скачиванием. +- Постоянный Docker volume для S3-хранилища. + +Сначала оригинал сохраняется в S3, затем в PostgreSQL фиксируются метаданные со статусом `ready`. При ошибке записи метаданных выполняется попытка удалить S3-объект. Полностью атомарной транзакции между PostgreSQL и S3 нет. Создание сообщения, привязка вложений и запись ключа идемпотентности выполняются в одной транзакции PostgreSQL. + +В `.env.example` используются параметры `POSTGRES_PASSWORD`, `SESSION_TTL_SECONDS`, `S3_ACCESS_KEY`, `S3_SECRET_KEY` и `S3_BUCKET`; внутренний `S3_ENDPOINT` задаётся в Compose. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Критерий | Команда / сценарий | Наблюдаемый результат | Доказательство | +|---|---|---|---| +| Публичный API | `make test` | PASS, включая сценарий вложений | `evidence/smoke.txt` | +| Две API-реплики | Проверка `X-Instance-Id` | Два разных UUID | `evidence/instances.txt` | +| Отказ API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS, 121 успешная запись, 0 ошибок | `evidence/failover.txt` | +| Восстановление записи | Failover-сценарий | 0,903 секунды | `evidence/failover.txt` | +| Сохранность данных | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS, подтверждённые данные и сессия сохранены | `evidence/persistence.txt` | +| PostgreSQL и S3 | `docker compose ps -a` | Хранилища запущены, API-реплики healthy | `evidence/containers.txt` | +| SQL-миграции | `docker compose run --rm --no-deps migrate` | Миграции применены | Журнал миграций | +| Реальный перезапуск хранилищ | Скрипт `restart.sh`, `docker inspect` | PostgreSQL и RustFS перезапущены | `evidence/restart-after.txt`, журнал действия | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Уровень 4 и трек 5 не заявлены. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Гипотеза:** при остановке одной API-реплики запросы продолжат обрабатываться второй, поскольку метаданные и оригиналы изображений находятся в общих хранилищах. После перезапуска PostgreSQL и RustFS данные должны сохраниться благодаря постоянным Docker volumes. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +**Среда:** локальный стенд Windows / WSL Ubuntu / Docker Desktop, Python 3.12, PostgreSQL 16, RustFS 1.0.3, две API-реплики, Nginx 1.29. + +Публичный smoke-тест завершился успешно: + +```text +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +``` + +Результаты failover-теста: + +```text +observed_instances_before: 2 +successful_writes: 121 +failed_attempts: 0 +first_recovery_seconds: 0.903 +PASS: failover preserved confirmed data and existing session. +``` + +Persistence-тест: + +```text +PASS: persistence preserved confirmed data and existing session. +``` + +| Воздействие | Ожидание | Результат | Восстановление / данные | +|---|---|---|---| +| Остановка одной API-реплики | Продолжение работы через вторую | PASS, 121 запись, 0 ошибок | 0,903 с, подтверждённые данные сохранены | +| Перезапуск PostgreSQL и RustFS | Сохранение данных и сессии | PASS | Потеря подтверждённых данных не выявлена | +| Загрузка и чтение PNG/JPEG | Совпадение оригинала и проверка прав доступа | Smoke PASS | Проверка целостности выполнена публичным сценарием | + +После проверок обе API-реплики восстановлены и имеют разные `X-Instance-Id`. + +Публичные тесты не являются полноценным нагрузочным испытанием. Отдельные измерения RPS, p50/p95/p99, CPU/RAM, точного профиля параллелизма и длительности прогрева не проводились. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Реализация выдерживает отказ одной API-реплики и перезапуск контейнеров с сохранением volumes. PostgreSQL, RustFS и Nginx остаются едиными точками отказа. + +Для хранения оригиналов применяется синхронная загрузка без фоновой обработки и генерации превью. При редком сбое между записью в S3 и фиксацией метаданных возможен временно неиспользуемый объект, поэтому в дальнейшем полезно добавить очистку таких объектов. + +Следующие улучшения: резервирование PostgreSQL и S3, мониторинг, резервное копирование и нагрузочные испытания. + +**Вывод:** реализовано общее S3-хранилище изображений для двух API-реплик. Публичный smoke, persistence и failover пройдены. При отказе реплики выполнена 121 успешная запись без ошибок, восстановление заняло 0,903 секунды. \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..6e5dd89d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,105 @@ + +services: + db: + image: postgres:16-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + + minio: + image: rustfs/rustfs:1.0.3 + command: ["/data"] + environment: + RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY in .env} + RUSTFS_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY in .env} + RUSTFS_ADDRESS: ":9000" + volumes: + - minio_data:/data + restart: unless-stopped + + s3-init: + build: . + command: ["python", "-m", "src.s3_setup"] + environment: &s3_env + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + depends_on: + minio: + condition: service_started + restart: "no" + + migrate: + build: . + command: ["python", "-m", "src.migrate"] + environment: + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + depends_on: + db: + condition: service_healthy + restart: "no" + + api1: + build: . + environment: + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + <<: *s3_env + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + interval: 5s + timeout: 3s + retries: 20 + stop_grace_period: 15s + restart: unless-stopped + + api2: + build: . + environment: + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@db:5432/messenger + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + <<: *s3_env + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + interval: 5s + timeout: 3s + retries: 20 + stop_grace_period: 15s + restart: unless-stopped + + nginx: + image: nginx:1.29-alpine + ports: + - "127.0.0.1:8080:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + api1: + condition: service_healthy + api2: + condition: service_healthy + restart: unless-stopped + +volumes: + postgres_data: + minio_data: diff --git a/evidence/containers.txt b/evidence/containers.txt new file mode 100644 index 00000000..39ac7127 --- /dev/null +++ b/evidence/containers.txt @@ -0,0 +1,8 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api1-1 mtusi2026lab4-api1 "uvicorn src.main:ap…" api1 8 minutes ago Up 15 seconds (healthy) 8080/tcp +mtusi2026lab4-api2-1 mtusi2026lab4-api2 "uvicorn src.main:ap…" api2 8 minutes ago Up 15 seconds (healthy) 8080/tcp +mtusi2026lab4-db-1 postgres:16-alpine "docker-entrypoint.s…" db 22 minutes ago Up 22 seconds (healthy) 5432/tcp +mtusi2026lab4-migrate-1 mtusi2026lab4-migrate "python -m src.migra…" migrate 4 minutes ago Exited (0) Less than a second ago +mtusi2026lab4-minio-1 rustfs/rustfs:1.0.3 "/entrypoint.sh /data" minio 22 minutes ago Up 22 seconds 9000-9001/tcp +mtusi2026lab4-nginx-1 nginx:1.29-alpine "/docker-entrypoint.…" nginx 22 minutes ago Up 2 minutes 127.0.0.1:8080->80/tcp +mtusi2026lab4-s3-init-1 mtusi2026lab4-s3-init "python -m src.s3_se…" s3-init 8 minutes ago Exited (0) 1 second ago diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..c8da41df --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 121, "failed_attempts": 0, "first_recovery_seconds": 0.903} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/instances.txt b/evidence/instances.txt new file mode 100644 index 00000000..656a3747 --- /dev/null +++ b/evidence/instances.txt @@ -0,0 +1,4 @@ +api1: +129285ef-99ed-475b-b17a-89a955089156 +api2: +ff71cea4-7b02-4f29-9931-b67b51594a6c diff --git a/evidence/migrations.txt b/evidence/migrations.txt new file mode 100644 index 00000000..9cef28ef --- /dev/null +++ b/evidence/migrations.txt @@ -0,0 +1,5 @@ +Already applied: 001_initial.sql +Already applied: 002_idempotency.sql +Already applied: 003_attachments.sql +Already applied: 004_allow_empty_message_text.sql +Migrations completed diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..793b001d --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/restart-after.txt b/evidence/restart-after.txt new file mode 100644 index 00000000..b5bd7bff --- /dev/null +++ b/evidence/restart-after.txt @@ -0,0 +1,4 @@ +/mtusi2026lab4-api1-1 2026-11-16T20:10:56.252894727Z +/mtusi2026lab4-api2-1 2026-11-16T20:10:56.255219312Z +/mtusi2026lab4-db-1 2026-11-16T20:10:49.101014076Z +/mtusi2026lab4-minio-1 2026-11-16T20:10:49.012555397Z diff --git a/evidence/restart-before.txt b/evidence/restart-before.txt new file mode 100644 index 00000000..89224254 --- /dev/null +++ b/evidence/restart-before.txt @@ -0,0 +1,3 @@ +/mtusi2026lab4-api1-1 2026-11-16T20:07:21.532908614Z +/mtusi2026lab4-db-1 2026-11-16T20:07:14.361368694Z +/mtusi2026lab4-minio-1 2026-11-16T20:07:14.222524846Z diff --git a/evidence/s3-access.txt b/evidence/s3-access.txt new file mode 100644 index 00000000..c3327f68 --- /dev/null +++ b/evidence/s3-access.txt @@ -0,0 +1,4 @@ +api1: +['messenger-attachments'] +api2: +['messenger-attachments'] diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..1e7b9ff4 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..0e4d5231 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,74 @@ + +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE, + display_name VARCHAR(100) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CONSTRAINT username_format + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT display_name_length + CHECK (char_length(display_name) BETWEEN 1 AND 100), + CONSTRAINT display_name_not_blank + CHECK (length(btrim(display_name)) > 0) +); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CONSTRAINT title_not_blank + CHECK (length(btrim(title)) > 0) +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL + REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL + REFERENCES users(id), + position INTEGER NOT NULL, + + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL + REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL + REFERENCES users(id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CONSTRAINT message_length + CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT message_not_blank + CHECK (length(btrim(text)) > 0) +); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash CHAR(64) PRIMARY KEY, + user_id UUID NOT NULL + REFERENCES users(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + revoked BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX IF NOT EXISTS idx_chat_members_user + ON chat_members(user_id, chat_id); + +CREATE INDEX IF NOT EXISTS idx_chats_order + ON chats(created_at, id); + +CREATE INDEX IF NOT EXISTS idx_messages_history + ON messages(chat_id, created_at, id); + +CREATE INDEX IF NOT EXISTS idx_sessions_user + ON sessions(user_id); + +CREATE INDEX IF NOT EXISTS idx_sessions_expiry + ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..663c6b6c --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,14 @@ + +CREATE TABLE IF NOT EXISTS message_idempotency ( + user_id UUID NOT NULL REFERENCES users(id), + chat_id UUID NOT NULL REFERENCES chats(id), + idempotency_key VARCHAR(128) NOT NULL, + request_hash CHAR(64) NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + PRIMARY KEY (user_id, chat_id, idempotency_key) +); + +CREATE INDEX IF NOT EXISTS idx_idempotency_created +ON message_idempotency(created_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..49e9f33e --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,34 @@ + +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + uploader_id UUID NOT NULL REFERENCES users(id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 CHAR(64) NOT NULL, + object_key TEXT NOT NULL UNIQUE, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + status TEXT NOT NULL DEFAULT 'ready', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT attachment_size_valid + CHECK (size_bytes BETWEEN 1 AND 10485760), + + CONSTRAINT attachment_dimensions_valid + CHECK (width > 0 AND height > 0), + + CONSTRAINT attachment_status_valid + CHECK (status IN ('ready', 'failed')) +); + +CREATE INDEX IF NOT EXISTS idx_attachments_chat +ON attachments(chat_id); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id), + attachment_id UUID NOT NULL REFERENCES attachments(id), + + PRIMARY KEY (message_id, attachment_id) +); diff --git a/migrations/004_allow_empty_message_text.sql b/migrations/004_allow_empty_message_text.sql new file mode 100644 index 00000000..9b721535 --- /dev/null +++ b/migrations/004_allow_empty_message_text.sql @@ -0,0 +1,13 @@ + +-- Lab4: messages with attachments may have empty text. +-- The API checks that a message contains text or attachments. + +ALTER TABLE messages +DROP CONSTRAINT IF EXISTS message_length; + +ALTER TABLE messages +DROP CONSTRAINT IF EXISTS message_not_blank; + +ALTER TABLE messages +ADD CONSTRAINT message_length +CHECK (char_length(text) <= 4000); diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..810ed1c8 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,37 @@ +events { + worker_connections 1024; +} + +http { + resolver 127.0.0.11 valid=2s ipv6=off; + resolver_timeout 2s; + + upstream messenger_api { + zone messenger_api 64k; + + server api1:8080 resolve max_fails=1 fail_timeout=2s; + server api2:8080 resolve max_fails=1 fail_timeout=2s; + } + + server { + client_max_body_size 11m; + listen 80; + + location / { + proxy_pass http://messenger_api; + + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_connect_timeout 2s; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + } + } +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..2cb25307 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.115.12 +uvicorn==0.34.2 +psycopg[binary]==3.2.6 +argon2-cffi==23.1.0 +boto3==1.40.70 +Pillow==11.2.1 \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..74de5913 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +echo "Stopping API replicas..." +docker compose stop api1 api2 + +echo "Restarting PostgreSQL and S3..." +docker compose restart db minio + +echo "Waiting for PostgreSQL..." +docker compose up -d --wait db + +echo "Checking S3 bucket..." +docker compose run --rm --no-deps s3-init + +echo "Starting API replicas..." +docker compose up -d --wait api1 api2 + +echo "Restart complete. Volumes preserved." diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..36b95cd6 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +if [[ -z "${TARGET_INSTANCE_ID:-}" ]]; then + echo "TARGET_INSTANCE_ID is not set" >&2 + exit 1 +fi + +for service in api1 api2; do + instance_id=$( + docker compose exec -T "$service" python -c ' +import urllib.request + +response = urllib.request.urlopen( + "http://127.0.0.1:8080/health/live", + timeout=2 +) + +print(response.headers.get("X-Instance-Id", "")) +' 2>/dev/null + ) || continue + + if [[ "$instance_id" == "$TARGET_INSTANCE_ID" ]]; then + echo "Stopping $service" + docker compose stop -t 2 "$service" + exit 0 + fi +done + +echo "Target API instance not found" >&2 +exit 1 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..54409a29 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,110 @@ + +import io +import re +import uuid +import hashlib + +from PIL import Image, UnidentifiedImageError + + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 + + +def valid_filename(filename): + if not isinstance(filename, str): + return False + + if not 1 <= len(filename) <= 255: + return False + + if filename.strip() == "": + return False + + if "/" in filename or "\\" in filename: + return False + + if any(ord(ch) < 32 or ord(ch) == 127 for ch in filename): + return False + + return True + + +def inspect_image(data, declared_mime): + if len(data) > MAX_BYTES: + raise ValueError("too_large") + + if not data: + raise ValueError("invalid_image") + + if declared_mime not in ("image/png", "image/jpeg"): + raise ValueError("invalid_image") + + try: + with Image.open(io.BytesIO(data)) as image: + image_format = image.format + + width, height = image.size + + if width * height > MAX_PIXELS: + raise ValueError("too_many_pixels") + + if image_format not in ("PNG", "JPEG"): + raise ValueError("invalid_image") + + actual_mime = ( + "image/png" + if image_format == "PNG" + else "image/jpeg" + ) + + if declared_mime != actual_mime: + raise ValueError("invalid_image") + + image.verify() + + except Image.DecompressionBombError: + raise ValueError("too_many_pixels") + except ( + UnidentifiedImageError, + OSError, + SyntaxError, + EOFError, + ): + raise ValueError("invalid_image") + + return { + "content_type": actual_mime, + "width": width, + "height": height, + "size_bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } + + +def attachment_json(row): + created_at = row["created_at"].astimezone( + __import__("datetime").timezone.utc + ).isoformat( + timespec="microseconds" + ).replace("+00:00", "Z") + + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["uploader_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": [], + "error_code": None, + "created_at": created_at, + "updated_at": created_at, + } + + + +def object_key(): + return f"originals/{uuid.uuid4()}" diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..e9dcd275 --- /dev/null +++ b/src/main.py @@ -0,0 +1,1142 @@ + +import base64 +import hashlib +import io +import json +import os +import re +import secrets +import uuid + +from datetime import datetime, timedelta, timezone + +import psycopg +from psycopg.rows import dict_row +from psycopg.errors import UniqueViolation + +from argon2 import PasswordHasher +from argon2.exceptions import ( + VerifyMismatchError, + VerificationError, + InvalidHashError, +) + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from fastapi.exceptions import RequestValidationError +from starlette.exceptions import HTTPException as StarletteHTTPException + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, UnidentifiedImageError + +app = FastAPI(title="MTUSI Messenger Lab4") + +INSTANCE_ID = str(uuid.uuid4()) + +DATABASE_URL = os.environ["DATABASE_URL"] +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) + +S3_BUCKET = os.environ["S3_BUCKET"] +MAX_IMAGE_BYTES = 10 * 1024 * 1024 +MAX_IMAGE_PIXELS = 40_000_000 + +s3 = boto3.client( + "s3", + endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_SECRET_KEY"], + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=3, + read_timeout=10, + retries={"max_attempts": 1}, + ), +) + +password_hasher = PasswordHasher() + + +# ---------- Helpers ---------- + +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def fail(status, code, message, headers=None): + raise ApiError(status, code, message, headers) + + +def db(): + return psycopg.connect(DATABASE_URL, row_factory=dict_row) + + +def utc_string(value): + return value.astimezone(timezone.utc).isoformat( + timespec="microseconds" + ).replace("+00:00", "Z") + + +def user_json(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": utc_string(row["created_at"]), + } + + + +def message_json(row, conn=None): + attachment_ids = [] + + if conn is not None: + attachments = conn.execute( + """ + SELECT attachment_id + FROM message_attachments + WHERE message_id = %s + ORDER BY attachment_id + """, + (row["id"],), + ).fetchall() + + attachment_ids = [ + str(item["attachment_id"]) + for item in attachments + ] + + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "attachment_ids": attachment_ids, + "created_at": utc_string(row["created_at"]), + } + + + +def chat_json(conn, row): + members = conn.execute( + """ + SELECT user_id FROM chat_members + WHERE chat_id = %s + ORDER BY position + """, + (row["id"],) + ).fetchall() + + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(m["user_id"]) for m in members], + "created_at": utc_string(row["created_at"]), + } + + +def valid_uuid(value): + if not isinstance(value, str): + return False + try: + return str(uuid.UUID(value)) == value + except ValueError: + return False + + +def valid_text(value, maximum): + return ( + isinstance(value, str) + and 1 <= len(value) <= maximum + and bool(value.strip()) + ) + + +def validate_limit(request): + value = request.query_params.get("limit", "50") + + if not re.fullmatch(r"[0-9]+", value): + fail(400, "invalid_request", "Invalid limit") + + number = int(value) + + if not 1 <= number <= 100: + fail(400, "invalid_request", "Invalid limit") + + return number + + +def make_cursor(resource, row): + payload = { + "resource": resource, + "created_at": utc_string(row["created_at"]), + "id": str(row["id"]), + } + + raw = json.dumps(payload).encode("utf-8") + + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def read_cursor(cursor, resource): + if not isinstance(cursor, str) or not cursor: + fail(400, "invalid_request", "Invalid cursor") + + try: + raw = base64.urlsafe_b64decode( + cursor + "=" * (-len(cursor) % 4) + ) + value = json.loads(raw) + + if value["resource"] != resource: + raise ValueError() + + if not valid_uuid(value["id"]): + raise ValueError() + + timestamp = datetime.fromisoformat( + value["created_at"].replace("Z", "+00:00") + ) + + if timestamp.tzinfo is None: + raise ValueError() + + return timestamp, value["id"] + + except (ValueError, KeyError, TypeError, UnicodeDecodeError): + fail(400, "invalid_request", "Invalid cursor") + + +async def json_body(request, required, allowed): + content_type = request.headers.get("content-type", "") + media_type = content_type.split(";")[0].strip().lower() + + if media_type != "application/json": + fail(415, "unsupported_media_type", "Expected JSON") + + try: + data = await request.json() + except Exception: + fail(400, "invalid_request", "Invalid JSON") + + if not isinstance(data, dict): + fail(400, "invalid_request", "Expected JSON object") + + if not set(required).issubset(data): + fail(400, "invalid_request", "Missing fields") + + if not set(data).issubset(allowed): + fail(400, "invalid_request", "Unexpected fields") + + return data + + +def unauthorized(kind="Bearer"): + if kind == "Basic": + header = 'Basic realm="messenger", charset="UTF-8"' + else: + header = 'Bearer realm="messenger"' + + fail( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": header}, + ) + + +def current_user(request): + authorization = request.headers.get("Authorization", "") + + if not authorization.startswith("Bearer "): + unauthorized() + + token = authorization[7:] + + if not token: + unauthorized() + + token_hash = hashlib.sha256(token.encode()).hexdigest() + + with db() as conn: + row = conn.execute( + """ + SELECT u.id, u.username, u.display_name, u.created_at + FROM sessions s + JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s + AND s.revoked = FALSE + AND s.expires_at > clock_timestamp() + """, + (token_hash,) + ).fetchone() + + if row is None: + unauthorized() + + return row + + +def accessible_chat(conn, chat_id, user_id): + if not valid_uuid(chat_id): + fail(400, "invalid_request", "Invalid chat ID") + + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = %s AND m.user_id = %s + """, + (chat_id, user_id) + ).fetchone() + + if row is None: + fail(404, "not_found", "Chat not found") + + return row + + +# ---------- Request IDs and errors ---------- + +def error_response(request, status, code, message, headers=None): + request_id = request.state.request_id + + response_headers = { + "X-Request-Id": request_id, + **(headers or {}), + } + + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers=response_headers, + ) + + +@app.middleware("http") +async def add_request_id(request, call_next): + request.state.request_id = str(uuid.uuid4()) + + try: + response = await call_next(request) + except psycopg.OperationalError: + response = error_response( + request, 503, "unavailable", "Database unavailable" + ) + except Exception: + response = error_response( + request, 500, "internal_error", "Internal server error" + ) + + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request, exc): + return error_response( + request, exc.status, exc.code, exc.message, exc.headers + ) + + +@app.exception_handler(StarletteHTTPException) +async def http_error_handler(request, exc): + return error_response( + request, exc.status_code, + "not_found" if exc.status_code == 404 else "invalid_request", + "Resource not found" if exc.status_code == 404 else "Invalid request", + ) + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(request, exc): + return error_response( + request, 400, "invalid_request", "Invalid request" + ) + + +# ---------- Health ---------- + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + + +@app.get("/health/ready") +def health_ready(): + try: + with db() as conn: + conn.execute("SELECT 1") + + s3.head_bucket(Bucket=S3_BUCKET) + + return {"status": "ok"} + + except ( + psycopg.Error, + BotoCoreError, + ClientError, + OSError, + ): + fail( + 503, + "unavailable", + "Storage unavailable", + ) + + +# ---------- Users ---------- + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + data = await json_body( + request, + required=["username", "display_name", "password"], + allowed=["username", "display_name", "password"], + ) + + username = data["username"] + display_name = data["display_name"] + password = data["password"] + + if ( + not isinstance(username, str) + or not re.fullmatch(r"[a-z0-9_]{3,32}", username) + or not valid_text(display_name, 100) + or not isinstance(password, str) + or not 12 <= len(password) <= 128 + ): + fail(400, "invalid_request", "Invalid user fields") + + password_hash = password_hasher.hash(password) + user_id = str(uuid.uuid4()) + + try: + with db() as conn: + row = conn.execute( + """ + INSERT INTO users + (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (user_id, username, display_name, password_hash) + ).fetchone() + + return user_json(row) + + except UniqueViolation: + fail(409, "conflict", "Username already exists") + + +@app.get("/api/v1/users/me") +def get_me(request: Request): + return user_json(current_user(request)) + + +# ---------- Sessions ---------- + +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request): + authorization = request.headers.get("Authorization", "") + + if not authorization.startswith("Basic "): + unauthorized("Basic") + + try: + encoded = authorization[6:] + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + + if ":" not in decoded: + raise ValueError() + + username, password = decoded.split(":", 1) + + except (ValueError, UnicodeDecodeError): + unauthorized("Basic") + + if not re.fullmatch(r"[a-z0-9_]{3,32}", username): + unauthorized("Basic") + + with db() as conn: + row = conn.execute( + "SELECT * FROM users WHERE username = %s", + (username,) + ).fetchone() + + if row is None: + unauthorized("Basic") + + try: + password_hasher.verify(row["password_hash"], password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + unauthorized("Basic") + + token = secrets.token_urlsafe(32) + token_hash = hashlib.sha256(token.encode()).hexdigest() + + expires_at = datetime.now(timezone.utc) + timedelta( + seconds=SESSION_TTL_SECONDS + ) + + with db() as conn: + conn.execute( + """ + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES (%s, %s, %s) + """, + (token_hash, row["id"], expires_at) + ) + + return { + "token": token, + "token_type": "Bearer", + "expires_at": utc_string(expires_at), + "user": user_json(row), + } + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def logout(request: Request): + current_user(request) + + token = request.headers["Authorization"][7:] + token_hash = hashlib.sha256(token.encode()).hexdigest() + + with db() as conn: + conn.execute( + "UPDATE sessions SET revoked = TRUE WHERE token_hash = %s", + (token_hash,) + ) + + return Response(status_code=204) + + +# ---------- Chats ---------- + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + actor = current_user(request) + + data = await json_body( + request, + required=["title", "member_ids"], + allowed=["title", "member_ids"], + ) + + title = data["title"] + member_ids = data["member_ids"] + + if not valid_text(title, 100): + fail(400, "invalid_request", "Invalid title") + + if ( + not isinstance(member_ids, list) + or not 1 <= len(member_ids) <= 100 + or not all(valid_uuid(x) for x in member_ids) + or len(member_ids) != len(set(member_ids)) + or str(actor["id"]) not in member_ids + ): + fail(400, "invalid_request", "Invalid members") + + chat_id = str(uuid.uuid4()) + + with db() as conn: + # All operations use one transaction. + existing = conn.execute( + """ + SELECT id FROM users + WHERE id = ANY(%s::uuid[]) + FOR KEY SHARE + """, + (member_ids,) + ).fetchall() + + if len(existing) != len(member_ids): + fail(400, "invalid_request", "Unknown member") + + row = conn.execute( + """ + INSERT INTO chats (id, title) + VALUES (%s, %s) + RETURNING id, title, created_at + """, + (chat_id, title) + ).fetchone() + + for position, member_id in enumerate(member_ids): + conn.execute( + """ + INSERT INTO chat_members (chat_id, user_id, position) + VALUES (%s, %s, %s) + """, + (chat_id, member_id, position) + ) + + result = chat_json(conn, row) + + return result + + +@app.get("/api/v1/chats") +def list_chats(request: Request): + actor = current_user(request) + limit = validate_limit(request) + cursor = request.query_params.get("cursor") + + resource = "chats:" + str(actor["id"]) + + with db() as conn: + if cursor is None: + rows = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = %s + ORDER BY c.created_at, c.id + LIMIT %s + """, + (actor["id"], limit + 1) + ).fetchall() + else: + timestamp, last_id = read_cursor(cursor, resource) + + rows = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = %s + AND (c.created_at, c.id) > (%s, %s) + ORDER BY c.created_at, c.id + LIMIT %s + """, + (actor["id"], timestamp, last_id, limit + 1) + ).fetchall() + + has_more = len(rows) > limit + page = rows[:limit] + + items = [chat_json(conn, row) for row in page] + + next_cursor = ( + make_cursor(resource, page[-1]) + if has_more else None + ) + + return {"items": items, "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, request: Request): + actor = current_user(request) + + with db() as conn: + row = accessible_chat(conn, chat_id, actor["id"]) + result = chat_json(conn, row) + + return result + + +# ---------- Attachments ---------- + +from src.attachments import ( + MAX_BYTES, + valid_filename, + inspect_image, + attachment_json, + object_key, +) + + +def attachment_for_user(conn, attachment_id, user_id): + if not valid_uuid(attachment_id): + fail(400, "invalid_request", "Invalid attachment ID") + + row = conn.execute( + """ + SELECT a.* + FROM attachments a + JOIN chat_members cm + ON cm.chat_id = a.chat_id + WHERE a.id = %s + AND cm.user_id = %s + """, + (attachment_id, user_id), + ).fetchone() + + if row is None: + fail(404, "not_found", "Attachment not found") + + return row + + +@app.post( + "/api/v1/chats/{chat_id}/attachments", + status_code=201, +) +async def upload_attachment(chat_id: str, request: Request): + actor = current_user(request) + + with db() as conn: + accessible_chat(conn, chat_id, actor["id"]) + + filename = request.query_params.get("filename") + + if not valid_filename(filename): + fail(400, "invalid_request", "Invalid filename") + + content_type = request.headers.get( + "content-type", "" + ).split(";")[0].strip().lower() + + if content_type not in ("image/png", "image/jpeg"): + fail( + 415, + "unsupported_media_type", + "Expected PNG or JPEG", + ) + + content_length = request.headers.get("content-length") + + if content_length is not None: + try: + declared_size = int(content_length) + except ValueError: + fail(400, "invalid_request", "Invalid Content-Length") + + if declared_size > MAX_BYTES: + fail( + 413, + "payload_too_large", + "Image exceeds 10 MiB", + ) + + # Bounded streaming: never read an unlimited request body. + data = bytearray() + + async for chunk in request.stream(): + if len(data) + len(chunk) > MAX_BYTES: + fail( + 413, + "payload_too_large", + "Image exceeds 10 MiB", + ) + data.extend(chunk) + + data = bytes(data) + + try: + metadata = inspect_image(data, content_type) + except ValueError as exc: + if str(exc) in ("too_large", "too_many_pixels"): + fail( + 413, + "payload_too_large", + "Image exceeds limits", + ) + + fail( + 415, + "unsupported_media_type", + "Invalid PNG or JPEG", + ) + + attachment_id = str(uuid.uuid4()) + key = object_key() + + try: + s3.put_object( + Bucket=S3_BUCKET, + Key=key, + Body=data, + ContentType=metadata["content_type"], + ) + except (BotoCoreError, ClientError, OSError): + fail( + 503, + "unavailable", + "Storage unavailable", + ) + + try: + with db() as conn: + # Check access again after the S3 operation. + accessible_chat(conn, chat_id, actor["id"]) + + row = conn.execute( + """ + INSERT INTO attachments + ( + id, chat_id, uploader_id, + filename, content_type, + size_bytes, sha256, + object_key, width, height, + status + ) + VALUES ( + %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, 'ready' + ) + RETURNING * + """, + ( + attachment_id, + chat_id, + actor["id"], + filename, + metadata["content_type"], + metadata["size_bytes"], + metadata["sha256"], + key, + metadata["width"], + metadata["height"], + ), + ).fetchone() + + except Exception: + # The DB operation failed. Avoid leaving an unused object. + try: + s3.delete_object(Bucket=S3_BUCKET, Key=key) + except Exception: + pass + raise + + return attachment_json(row) + + +@app.get("/api/v1/attachments/{attachment_id}") +def get_attachment(attachment_id: str, request: Request): + actor = current_user(request) + + with db() as conn: + row = attachment_for_user( + conn, attachment_id, actor["id"] + ) + + return attachment_json(row) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + attachment_id: str, + request: Request, +): + actor = current_user(request) + + variant = request.query_params.get("variant", "original") + + with db() as conn: + row = attachment_for_user( + conn, attachment_id, actor["id"] + ) + + if row["status"] != "ready": + fail(409, "conflict", "Attachment is not ready") + + if variant != "original": + fail(404, "not_found", "Variant not found") + + try: + result = s3.get_object( + Bucket=S3_BUCKET, + Key=row["object_key"], + ) + body_stream = result["Body"] + + try: + data = body_stream.read(MAX_BYTES + 1) + finally: + body_stream.close() + + except (BotoCoreError, ClientError, OSError): + fail( + 503, + "unavailable", + "Storage unavailable", + ) + + if ( + len(data) != row["size_bytes"] + or hashlib.sha256(data).hexdigest() != row["sha256"] + ): + fail( + 503, + "unavailable", + "Stored object is unavailable", + ) + + return Response( + content=data, + media_type=row["content_type"], + headers={ + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + }, + ) + +def get_idempotency_key(request): + key = request.headers.get("Idempotency-Key") + + if ( + key is None + or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key) + ): + fail( + 400, + "invalid_request", + "Invalid or missing Idempotency-Key" + ) + + return key + + +def request_fingerprint(data): + normalized = { + "text": data["text"], + "attachment_ids": data.get("attachment_ids", []), + } + + encoded = json.dumps( + normalized, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + return hashlib.sha256(encoded).hexdigest() + + + +def idempotency_lock_id(user_id, chat_id, key): + scope = f"{user_id}:{chat_id}:{key}" + + digest = hashlib.sha256( + scope.encode("utf-8") + ).digest() + + return int.from_bytes( + digest[:8], + byteorder="big", + signed=True + ) + + +# ---------- Messages ---------- + + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def send_message(chat_id: str, request: Request): + actor = current_user(request) + + data = await json_body( + request, + required=["text"], + allowed=["text", "attachment_ids"], + ) + + message_text = data["text"] + attachment_ids = data.get("attachment_ids", []) + + if ( + not isinstance(message_text, str) + or len(message_text) > 4000 + ): + fail(400, "invalid_request", "Invalid message text") + + if ( + not isinstance(attachment_ids, list) + or len(attachment_ids) > 10 + or not all(valid_uuid(x) for x in attachment_ids) + or len(attachment_ids) != len(set(attachment_ids)) + ): + fail(400, "invalid_request", "Invalid attachment_ids") + + if not message_text.strip() and not attachment_ids: + fail(400, "invalid_request", "Empty message") + + key = get_idempotency_key(request) + fingerprint = request_fingerprint(data) + user_id = actor["id"] + + with db() as conn: + accessible_chat(conn, chat_id, user_id) + + lock_id = idempotency_lock_id( + user_id, chat_id, key + ) + + conn.execute( + "SELECT pg_advisory_xact_lock(%s)", + (lock_id,), + ) + + conn.execute( + """ + DELETE FROM message_idempotency + WHERE user_id = %s + AND chat_id = %s + AND idempotency_key = %s + AND created_at < + clock_timestamp() - INTERVAL '24 hours' + """, + (user_id, chat_id, key), + ) + + previous = conn.execute( + """ + SELECT i.request_hash, m.* + FROM message_idempotency i + JOIN messages m ON m.id = i.message_id + WHERE i.user_id = %s + AND i.chat_id = %s + AND i.idempotency_key = %s + """, + (user_id, chat_id, key), + ).fetchone() + + if previous is not None: + if previous["request_hash"] != fingerprint: + fail( + 409, + "conflict", + "Idempotency-Key already used " + "with different content", + ) + + return message_json(previous, conn) + + for attachment_id in attachment_ids: + attachment = conn.execute( + """ + SELECT id, status + FROM attachments + WHERE id = %s + AND chat_id = %s + FOR KEY SHARE + """, + (attachment_id, chat_id), + ).fetchone() + + if attachment is None: + fail( + 404, + "not_found", + "Attachment not found", + ) + + if attachment["status"] != "ready": + fail( + 409, + "conflict", + "Attachment is not ready", + ) + + message_id = str(uuid.uuid4()) + + row = conn.execute( + """ + INSERT INTO messages + (id, chat_id, sender_id, text) + VALUES (%s, %s, %s, %s) + RETURNING id, chat_id, sender_id, + text, created_at + """, + (message_id, chat_id, user_id, message_text), + ).fetchone() + + for attachment_id in attachment_ids: + conn.execute( + """ + INSERT INTO message_attachments + (message_id, attachment_id) + VALUES (%s, %s) + """, + (message_id, attachment_id), + ) + + conn.execute( + """ + INSERT INTO message_idempotency + ( + user_id, chat_id, idempotency_key, + request_hash, message_id + ) + VALUES (%s, %s, %s, %s, %s) + """, + ( + user_id, chat_id, key, + fingerprint, message_id, + ), + ) + + return message_json(row, conn) + + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: str, request: Request): + actor = current_user(request) + limit = validate_limit(request) + cursor = request.query_params.get("cursor") + + resource = "messages:" + chat_id + + with db() as conn: + accessible_chat(conn, chat_id, actor["id"]) + + if cursor is None: + rows = conn.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s + ORDER BY created_at, id + LIMIT %s + """, + (chat_id, limit + 1) + ).fetchall() + else: + timestamp, last_id = read_cursor(cursor, resource) + + rows = conn.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s + AND (created_at, id) > (%s, %s) + ORDER BY created_at, id + LIMIT %s + """, + (chat_id, timestamp, last_id, limit + 1) + ).fetchall() + + has_more = len(rows) > limit + page = rows[:limit] + + next_cursor = ( + make_cursor(resource, page[-1]) + if has_more else None + ) + + items = [message_json(row, conn) for row in page] + + return {"items": items, "next_cursor": next_cursor} diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..c13aa99c --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,53 @@ + +import os +from pathlib import Path + +import psycopg + + +DATABASE_URL = os.environ["DATABASE_URL"] +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" + + +def run_migrations(): + with psycopg.connect(DATABASE_URL) as conn: + with conn.cursor() as cur: + # Prevent simultaneous migration runs. + cur.execute("SELECT pg_advisory_xact_lock(2026002)") + + cur.execute(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL + DEFAULT clock_timestamp() + ) + """) + + for file in sorted(MIGRATIONS_DIR.glob("*.sql")): + cur.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (file.name,) + ) + + if cur.fetchone(): + print(f"Already applied: {file.name}") + continue + + sql = file.read_text(encoding="utf-8") + + for statement in sql.split(";"): + if statement.strip(): + cur.execute(statement) + + cur.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (file.name,) + ) + + print(f"Applied migration: {file.name}") + + print("Migrations completed") + + +if __name__ == "__main__": + run_migrations() diff --git a/src/s3_setup.py b/src/s3_setup.py new file mode 100644 index 00000000..7798af60 --- /dev/null +++ b/src/s3_setup.py @@ -0,0 +1,60 @@ + +import os +import time + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + + +def main(): + bucket = os.environ["S3_BUCKET"] + + client = boto3.client( + "s3", + endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_SECRET_KEY"], + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + ), + ) + + for attempt in range(30): + try: + client.head_bucket(Bucket=bucket) + print(f"Bucket already exists: {bucket}", flush=True) + break + + except ClientError as exc: + code = exc.response["Error"]["Code"] + + if code in ("404", "NoSuchBucket"): + try: + client.create_bucket(Bucket=bucket) + print(f"Bucket created: {bucket}", flush=True) + break + except Exception as create_error: + print( + f"Waiting for S3: {create_error}", + flush=True, + ) + else: + print(f"Waiting for S3: {exc}", flush=True) + + except Exception as exc: + print(f"Waiting for S3: {exc}", flush=True) + + if attempt == 29: + raise RuntimeError("Cannot initialize S3 bucket") + + time.sleep(2) + + # No public bucket policy is configured. + print(f"Private S3 bucket is ready: {bucket}", flush=True) + + +if __name__ == "__main__": + main()