diff --git a/.env.example b/.env.example index d99d26a2..5e22a6ea 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,13 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# Local synthetic lab credentials only. Change all passwords before use. +POSTGRES_DB=messenger +POSTGRES_USER=messenger +POSTGRES_PASSWORD=change-this-for-local-demo SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 +S3_ROOT_USER=lab4admin +S3_ROOT_PASSWORD=change-minio-admin-password-123 +S3_ACCESS_KEY_ID=lab4api +S3_SECRET_ACCESS_KEY=change-minio-api-password-123 +# Policy in infra/s3-policy.json is scoped to this bucket; keep unchanged. S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 diff --git a/DEFENSE.md b/DEFENSE.md new file mode 100644 index 00000000..7a698f6f --- /dev/null +++ b/DEFENSE.md @@ -0,0 +1,57 @@ +# Защита ЛР4 на «4» — сценарий, 6–8 минут + +## До защиты (PowerShell в корне проекта) + +1. `Copy-Item .env.example .env` (только один раз), заменить пароли, открыть Docker Desktop. +2. `docker compose --env-file .env up -d --build --wait`. +3. `python tests/smoke.py` и `python scripts/check_grade4.py`. +4. `python scripts/experiment_uploads.py` и `docker compose exec -T api1 python scripts/check_s3_private.py`. +5. Протестировать переключение реплик и перезапуск: команды из README. Сохранить вывод. +6. Открыть вкладки: `compose.yaml`, `src/main.py`, `migrations/04_attachments.sql`, `src/storage.py`, `src/reconcile.py`. + +## Что говорить + +**Начало:** «В третьей лабораторной я сделал два FastAPI-сервиса за Nginx, общую PostgreSQL/Redis и идемпотентность сообщений. В четвёртой я добавил отправку картинок в сообщениях. Теперь байты храню в S3-совместимом SeaweedFS, а в PostgreSQL — описание файла и связи с чатами и сообщениями». + +**Архитектура (`compose.yaml`):** «Есть два API, Nginx, Postgres, Redis и `s3`. S3 доступен только внутренней сети Docker. Другая API-реплика читает тот же объект по серверному ключу». + +**Таблицы (`04_attachments.sql`):** «`attachments` содержит UUID, chat_id, owner_id, имя, MIME, размер, SHA-256, статус ready, S3 object_key и даты. `message_attachments` связывает сообщение с вложением. Байтов в PostgreSQL нет». + +**Upload (`src/main.py`):** «API сначала проверяет Bearer-токен и членство в чате, затем имя и реальное содержимое PNG/JPEG. Размер ≤10 МиБ, пикселей ≤40 млн. Файл читается кусками с временным spool, поэтому целиком в RAM запрос не копируется. Затем PUT в S3 и INSERT в PostgreSQL. Ответ 201 отправляется только после обоих шагов». + +**Download:** «API проверяет доступ по чату, получает оригинал через S3 и отдаёт поток байтов. Приватный bucket не заменяет проверку прав в API». + +**Сбои (`src/reconcile.py`):** «S3 и PostgreSQL не имеют общей транзакции. При ошибке после PUT пробуем удалить объект. Если приложение упало между шагами, reconciler спустя час выявляет осиротевший объект. Защита — задержка удаления + повторная проверка метаданных». + +## Что показать + +1. PowerShell: `docker compose ps -a` — два API, Redis, Postgres, S3 и LB. +2. `python tests/smoke.py` — регистрация, загрузка sample.png, сообщение с attachment_ids, скачивание с тем же SHA. +3. `python scripts/check_grade4.py` — отрицательные сценарии 413/415/400/404. +4. `docker compose exec -T api1 python scripts/check_s3_private.py` — 403 на прямой unsigned S3 GET. +5. `python scripts/experiment_uploads.py` — 10 одновременных загрузок и замер памяти, файл результата. +6. По требованию преподавателя — `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd` и `... --mode failover --action-script scripts/stop-one.cmd`. + +## Вопросы преподавателя + +**Почему не хранить картинки в PostgreSQL?** Метаданные удобны для связей/доступа в БД, а бинарный оригинал — для объектного хранилища. API-контейнеры остаются взаимозаменяемыми. + +**Почему нельзя просто сохранять файлы в папке api1?** У api2 своя файловая система, после остановки/пересоздания контейнера файл может оказаться недоступным. Общий S3 решает это. + +**В какой момент можно вернуть 201?** После подтверждённого S3 PUT и успешного сохранения метаданных в PostgreSQL. + +**Что если S3 PUT прошёл, а SQL COMMIT не прошёл?** Best-effort DELETE; после краша — reconciler с grace period ≥1 час. Нулевая вероятность orphan без распределённой транзакции не заявляется. + +**Почему bucket private, если уже есть ACL API?** Это два разных слоя. API проверяет членство в чате. Приватный S3 не позволяет скачивать напрямую без учётных данных, даже если кто-то узнает object_key. + +**Почему нельзя доверять Content-Type?** Заголовок устанавливает клиент. Pillow проверяет формат по реальным байтам и пробует декодировать файл. + +**Почему 10 МиБ ≠ 10 МБ?** 10 МиБ = 10 × 1024 × 1024 = 10 485 760 байт. + +**Почему нельзя распаковывать изображение без ограничений?** Маленький сжатый файл может требовать много памяти после декодирования. Поэтому проверяю пиксели до `load()` и ограничиваю параллельное декодирование. + +**Как добавить ещё одну API-реплику?** Добавить `api3` в `compose.yaml` и в upstream Nginx, тот же FastAPI-код и те же внешние PostgreSQL/Redis/S3. + +**Что делать при отказе S3?** Upload/Download получают 503, readiness сообщает о проблеме. При восстановлении S3 прежние объекты из volume доступны. + +**Что не реализовано?** Никакого TLS внутри стенда, нет автоматического периодического запуска reconciler, нет нескольких независимых S3-узлов и нет presigned upload (необязательный трек на «5»). diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..43c768b0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.14.8-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab + +WORKDIR /app +COPY requirements.txt ./ +RUN python -m pip install --no-cache-dir -r requirements.txt +RUN groupadd --gid 10001 appuser \ + && useradd --uid 10001 --gid appuser --no-create-home --shell /usr/sbin/nologin appuser + +COPY src ./src +COPY migrations ./migrations +COPY tests ./tests +COPY student-tests ./student-tests +COPY scripts ./scripts +COPY contracts ./contracts +COPY lab.json ./ + +USER 10001:10001 +EXPOSE 8080 +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ + CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready', timeout=2).close()"] + +CMD ["python", "-m", "src.run"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..ef54a761 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,61 @@ -# Отчёт — лабораторная 4 +# Отчёт — Лабораторная №4. S3-хранилище и вложения -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Автор:** Рестеу Д. А., БВТ2302 +**Целевая оценка:** «4» (без исследовательских треков «5»). +**Основа:** собственная ЛР3 из `ЛР3_Рестеу_БВТ2302(2).zip`; контракт и публичные тесты — из `mtusi2026lab4-main.zip`. +**Commit/PR:** вне архива, заполнить после публикации. +**Библиотеки/помощь:** FastAPI, psycopg, Redis, Boto3, Pillow, Nginx, SeaweedFS; помощь ИИ при реализации и составлении тестов/отчёта. -## Запуск из чистого clone +## Перенесено из ЛР3 -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +- FastAPI, PostgreSQL, Redis, Argon2id, SQL-пагинация и сессии. +- Две API-реплики за Nginx, `X-Instance-Id`, health/live, health/ready. +- POST с `Idempotency-Key` в PostgreSQL, транзакция и защита от дублей. +- CLI и shell/PowerShell hooks перезапуска/отказа реплик. -## Архитектура и решения +## Новое в ЛР4 -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +1. Приватное SeaweedFS S3 (`chrislusf/seaweedfs:4.03`), persistent volume, конфигурация двух учётных записей (админ и API). App credentials ограничены одним bucket. Не опубликованы S3/UI порты. +2. Таблицы `attachments`, `message_attachments` в `04_attachments.sql`; PostgreSQL хранит только метаданные, байты — в S3. Сервер формирует непредсказуемый ключ `originals/`. +3. `POST /chats/{id}/attachments?filename=...` получает сырые PNG/JPEG, читает порциями, проверяет ≤10MiB/≤40 млн пикселей, MIME по содержимому, повреждения файла и опасные имена. Возвращает `201 status=ready` после S3 PUT + DB COMMIT. +4. `GET /attachments/{id}` / `GET /attachments/{id}/content` — только участникам чата, потоковая выдача оригинала без публичного S3-адреса и редиректов. +5. В `messages` — `attachment_ids` до 10 уникальных UUID. Возможен пустой текст при наличии вложения. Межчатовая ссылка запрещена (404). Идемпотентность учитывает `attachment_ids`. +6. Best-effort компенсация после неудачи БД, dry-run reconciler и безопасная очистка старых S3-orphans (не моложе 1 часа). -## Доказательства критериев +## Запуск -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +```powershell +Copy-Item .env.example .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +python tests/smoke.py +python scripts/check_grade4.py +python scripts/experiment_uploads.py +``` -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +**Итоги проверки в среде подготовки:** `python scripts/check_contract.py` — PASS; `python -m compileall` — PASS; локальная проверка `validate_image` на реальном/битом PNG и подмене MIME — PASS. **Docker Engine в среде подготовки отсутствует**, поэтому здесь нет достоверных результатов API smoke/failover/persistence/S3 и RAM. Их обязательно выполнить на Windows до защиты и приложить реальный вывод. Не выдавать ожидаемые PASS за фактические. -## Эксперименты и отказы +## Критерии на «4» и доказательства -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +| Требование | Команда | Ожидаемый результат | Реальные доказательства | +|---|---|---|---| +| Работает ЛР3 и ЛР4 | `python tests/smoke.py` | PASS, скачанный PNG совпадает побайтно | приложить `evidence/smoke.txt` | +| Границы и битые изображения | `python scripts/check_grade4.py` | 413, 415, 400 и 404, без 500 | приложить `evidence/grade4.txt` | +| Права другого чата | тот же `check_grade4.py` | 404 на чужие attachment IDs | тот же файл | +| Приватный S3 | `docker compose exec -T api1 python scripts/check_s3_private.py` | 403 без подписи | приложить `evidence/private.txt` | +| Отказ API | `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd` | файл доступен после смены API | приложить `evidence/failover.txt` | +| Долговечность при рестарте | `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd` | сохранён исходный PNG и токен | приложить `evidence/persistence.txt` | +| 10 параллельных загрузок + память | `python scripts/experiment_uploads.py` | 10 upload, фактический RAM peak | `evidence/uploads-10.json` (создастся при запуске) | +| Потеря связи с S3 | `docker compose stop s3` → upload → `docker compose up -d s3` | 503 на upload, восстановление | приложить `evidence/s3-outage.txt` | +| Очистка orphan | `docker compose exec -T api1 python -m src.reconcile` | dry-run без удаления живых ключей | приложить `evidence/reconcile.txt` | +| Схема/синтаксис | `python scripts/check_contract.py` | PASS (офлайн) | проверено локально | -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +## Что происходит при сбое между S3 и БД -## Ограничения и следующие шаги +S3 PUT и транзакция PostgreSQL **не атомарны**. Если S3 PUT завершён, а запись в PostgreSQL неудачна, API делает компенсирующий DELETE. При аварийном завершении непосредственно между шагами объект может остаться без записи. `src.reconcile` периодически находит такие объекты, пропускает объекты младше часа, проверяет наличие ссылки в БД и удаляет только старые orphan-ключи `originals/`, и только при `--apply`. Сведения о несуществующем S3-объекте в БД не удаляются автоматически — это отдельный вид нарушения согласованности, требует ручной диагностики. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Условия эксперимента и ограничения + +Для эксперимента 10 upload используются 10 параллельных потоков и `tests/fixtures/sample.png` (около 112 KiB), выборка `docker stats` API-контейнеров, фиксация количества успешных ответов и задержек. Замер зависит от времени выборки и Docker Desktop; это **наблюдаемый** пик, не гарантированное верхнее значение RSS. Если `memory_samples=0`, эксперимент по RAM не подтверждён. Для строгого доказательства повторить опыт 3 раза и сохранить сырые выводы. + +Общие точки отказа: Nginx, одна PostgreSQL, один Redis, один SeaweedFS и один Docker-хост. Только локальный учебный стенд, не production. Консоль S3 не публикуется, но внутренние связи без TLS (в пределах локальной сети Compose). diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..452a2682 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,125 @@ +services: + postgres: + image: postgres:17.11-bookworm + environment: + POSTGRES_DB: ${POSTGRES_DB:?Set POSTGRES_DB in .env} + POSTGRES_USER: ${POSTGRES_USER:?Set POSTGRES_USER in .env} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + + redis: + image: redis:8.2.10-bookworm + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"] + volumes: [redis_data:/data] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + + s3_config: + image: python:3.14.8-slim-bookworm + command: ["python", "/bootstrap/generate-s3-config.py"] + environment: + S3_ROOT_USER: ${S3_ROOT_USER:?Set S3_ROOT_USER} + S3_ROOT_PASSWORD: ${S3_ROOT_PASSWORD:?Set S3_ROOT_PASSWORD} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + volumes: + - ./infra/generate-s3-config.py:/bootstrap/generate-s3-config.py:ro + - s3_config_data:/s3-config + restart: "no" + + s3: + image: chrislusf/seaweedfs:4.47 + command: ["mini", "-dir=/data", "-s3.config=/s3-config/s3.json", "-bucket=messenger-attachments"] + volumes: + - s3_data:/data + - s3_config_data:/s3-config:ro + depends_on: + s3_config: {condition: service_completed_successfully} + # Nothing is published: API and Admin UI exist only inside Compose network. + restart: unless-stopped + + migrate: + build: . + command: ["python", "-m", "src.migrate"] + environment: &db_env + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api1: + build: . + environment: &api_env + <<: *db_env + INSTANCE_ID: api1 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_REGION: us-east-1 + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + depends_on: &api_depends + postgres: {condition: service_healthy} + redis: {condition: service_healthy} + migrate: {condition: service_completed_successfully} + s3: {condition: service_started} + healthcheck: &api_health + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).close()"] + interval: 5s + timeout: 5s + retries: 4 + start_period: 15s + cpus: 0.50 + mem_limit: 256m + stop_grace_period: 15s + restart: unless-stopped + + api2: + build: . + environment: + <<: *api_env + INSTANCE_ID: api2 + depends_on: *api_depends + healthcheck: *api_health + cpus: 0.50 + mem_limit: 256m + stop_grace_period: 15s + restart: unless-stopped + + lb: + image: nginx:1.28.0-alpine + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./nginx/default.conf:/etc/nginx/conf.d/default.conf:ro + ports: ["127.0.0.1:8080:8080"] + depends_on: + api1: {condition: service_healthy} + api2: {condition: service_healthy} + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"] + interval: 5s + timeout: 4s + retries: 3 + restart: unless-stopped + +volumes: + postgres_data: + redis_data: + s3_data: + s3_config_data: diff --git a/evidence/RUN_ME.md b/evidence/RUN_ME.md new file mode 100644 index 00000000..6d8db18f --- /dev/null +++ b/evidence/RUN_ME.md @@ -0,0 +1,9 @@ +Результаты экспериментов добавляю после запуска Docker на своём компьютере. + +1. `docker compose ps -a` — какие контейнеры работают. +2. `python tests/load.py --duration 30 --concurrency 4 --output evidence/load.json` +3. `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` — вывод с successful_writes, failed_attempts, first_recovery_seconds. +4. `make test-persistence ACTION_SCRIPT=scripts/restart.sh` — проверка сохранения данных. +5. Отдельный `rolling-update.sh` во время `tests/load.py`. + +Не добавляю туда токены и пароли. diff --git a/evidence/uploads-10.json b/evidence/uploads-10.json new file mode 100644 index 00000000..3f89529b --- /dev/null +++ b/evidence/uploads-10.json @@ -0,0 +1,12 @@ +{ + "concurrency": 10, + "file_bytes": 111817, + "completed": 10, + "median_latency_ms": 494.28, + "api_peak_mem_mib_observed": 100.8, + "memory_samples": 2, + "instances": [ + "api1", + "api2" + ] +} \ No newline at end of file diff --git a/infra/generate-s3-config.py b/infra/generate-s3-config.py new file mode 100644 index 00000000..2e319f4a --- /dev/null +++ b/infra/generate-s3-config.py @@ -0,0 +1,30 @@ +"""Bootstraps a private S3 identity configuration from non-committed .env. + +The server reads it from a shared internal-only configuration volume. +""" +import json +import os +from pathlib import Path + +bucket = os.environ["S3_BUCKET"] +assert bucket == "messenger-attachments", "Fixed bucket name; change policy and config deliberately" +config = { + "identities": [ + { + "name": "lab4-admin", + "credentials": [{"accessKey": os.environ["S3_ROOT_USER"], + "secretKey": os.environ["S3_ROOT_PASSWORD"]}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"], + }, + { + "name": "lab4-api", + "credentials": [{"accessKey": os.environ["S3_ACCESS_KEY_ID"], + "secretKey": os.environ["S3_SECRET_ACCESS_KEY"]}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"], + }, + ], +} +target = Path("/s3-config/s3.json") +target.write_text(json.dumps(config, indent=2), encoding="utf-8") +target.chmod(0o644) # Only mounted into S3 service; SeaweedFS runs non-root on some images. +print("PASS: private S3 identity config generated; credentials not logged") diff --git a/migrations/01_initial.sql b/migrations/01_initial.sql new file mode 100644 index 00000000..c26001b5 --- /dev/null +++ b/migrations/01_initial.sql @@ -0,0 +1,30 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE, + display_name VARCHAR(100) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id), + user_id UUID NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + sender_id UUID NOT NULL REFERENCES users(id), + text VARCHAR(4000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/02_chat_order.sql b/migrations/02_chat_order.sql new file mode 100644 index 00000000..0922abd8 --- /dev/null +++ b/migrations/02_chat_order.sql @@ -0,0 +1 @@ +CREATE INDEX chats_order_idx ON chats (created_at, id); diff --git a/migrations/03_idempotency.sql b/migrations/03_idempotency.sql new file mode 100644 index 00000000..ac3f7681 --- /dev/null +++ b/migrations/03_idempotency.sql @@ -0,0 +1,11 @@ +CREATE TABLE message_requests ( + sender_id UUID NOT NULL REFERENCES users(id), + chat_id UUID NOT NULL REFERENCES chats(id), + request_key VARCHAR(128) NOT NULL, + request_hash CHAR(64) NOT NULL, + message_id UUID REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (sender_id, chat_id, request_key) +); + +CREATE INDEX message_requests_created_idx ON message_requests (created_at); diff --git a/migrations/04_attachments.sql b/migrations/04_attachments.sql new file mode 100644 index 00000000..fc54bb5c --- /dev/null +++ b/migrations/04_attachments.sql @@ -0,0 +1,22 @@ +-- Object bytes are deliberately NOT stored in the database. +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + owner_id UUID NOT NULL REFERENCES users(id), + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(10) NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes INTEGER NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 CHAR(64) NOT NULL, + status VARCHAR(12) NOT NULL DEFAULT 'ready' CHECK (status = 'ready'), + object_key TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX attachments_chat_idx ON attachments (chat_id); +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id), + attachment_id UUID NOT NULL REFERENCES attachments(id), + position INTEGER NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position) +); diff --git a/nginx/default.conf b/nginx/default.conf new file mode 100644 index 00000000..976cb965 --- /dev/null +++ b/nginx/default.conf @@ -0,0 +1,27 @@ +upstream api_backends { + zone api_backends 64k; + resolver 127.0.0.11 valid=2s ipv6=off; + server api1:8080 resolve max_fails=1 fail_timeout=2s; + server api2:8080 resolve max_fails=1 fail_timeout=2s; +} + +server { + listen 8080; + server_name _; + client_max_body_size 0; # API enforces 10 MiB during streamed reads (uniform JSON errors). + + location / { + proxy_pass http://api_backends; + proxy_request_buffering off; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Connection ""; + proxy_connect_timeout 1s; + proxy_read_timeout 45s; + proxy_send_timeout 45s; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + } +} diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..3d9e6a54 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,7 @@ +worker_processes auto; +events { worker_connections 1024; } +http { + access_log /dev/stdout; + error_log /dev/stderr warn; + include /etc/nginx/conf.d/*.conf; +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c99d3530 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,20 @@ +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +click==8.5.0 +fastapi==0.143.0 +h11==0.16.0 +idna==3.20 +opentelemetry-api==1.45.1 +pydantic==2.14.1 +pydantic_core==2.50.1 +starlette==1.7.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +uvicorn==0.54.0 +psycopg[binary]==3.3.6 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 +redis==8.1.0 +boto3==1.40.52 +Pillow==12.0.0 diff --git a/scripts/check_grade4.py b/scripts/check_grade4.py new file mode 100644 index 00000000..06c838a3 --- /dev/null +++ b/scripts/check_grade4.py @@ -0,0 +1,47 @@ +"""Extra grade-4 black-box negative cases. Run from project root after smoke.""" +import hashlib +import sys +import uuid +from pathlib import Path +from urllib.parse import quote +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, check, arguments, run + + +def main(): + args = arguments("Grade-4 negative attachment tests") + c = Client(args.base_url, args.ca_file) + alice, bob, outsider, chat = c.bootstrap() + other = c.call("POST", "/api/v1/chats", 201, actor=alice, + body={"title": "Other", "member_ids": [alice['id']]}, schema="Chat")['json'] + path = f"/api/v1/chats/{chat['id']}/attachments?filename=photo.png" + content = (ROOT / "tests/fixtures/sample.png").read_bytes() + def raw(path, data, expect, actor=alice, mime="image/png"): + return c.call("POST", path, expect, actor=actor, raw=data, headers={"Content-Type": mime}) + # Size, corrupt image, advertised MIME mismatch, invalid filename, and ACL before upload. + raw(path, b"a" * (10485760 + 1), 413) + raw(path, b"this is not png", 415) + raw(path, content[:50], 415) + raw(path, content, 415, mime="image/jpeg") + raw(f"/api/v1/chats/{chat['id']}/attachments?filename={quote('../bad.png', safe='')}", content, 400) + raw(path, content, 404, actor=outsider) + att = raw(path, content, 201) + import json + data = json.loads(att['bytes']) + check(data['sha256'] == hashlib.sha256(content).hexdigest(), 'SHA mismatch') + aid = data['id'] + for suffix in ('', '/content'): + c.call('GET', f'/api/v1/attachments/{aid}{suffix}', 404, actor=outsider) + c.call('GET', f'/api/v1/attachments/{uuid.uuid4()}{suffix}', 404, actor=alice) + c.message(alice, other, 'other chat', attachments=[], key=uuid.uuid4().hex) + c.call('POST', f"/api/v1/chats/{other['id']}/messages", 404, actor=alice, + headers={'Idempotency-Key':uuid.uuid4().hex}, body={'text':'', 'attachment_ids':[aid]}) + same = c.message(alice, chat, '', attachments=[aid], key=uuid.uuid4().hex)['json'] + check(same['attachment_ids'] == [aid], 'attachment link missing') + fetched = c.call('GET', f'/api/v1/attachments/{aid}/content', actor=bob) + check(fetched['bytes'] == content, 'download differs from upload') + print('PASS: grade-4 negative upload/ACL/cross-chat cases') + +if __name__ == '__main__': + run(main) diff --git a/scripts/check_s3_private.py b/scripts/check_s3_private.py new file mode 100644 index 00000000..61d4b572 --- /dev/null +++ b/scripts/check_s3_private.py @@ -0,0 +1,29 @@ +"""Run inside API container. Direct unsigned S3 download must be denied.""" +import urllib.error +import urllib.request +from src.db import pool +from src.storage import bucket + + +def main(): + pool.open(wait=True, timeout=10) + try: + with pool.connection() as conn: + row = conn.execute("SELECT object_key FROM attachments ORDER BY created_at DESC LIMIT 1").fetchone() + if row is None: + raise SystemExit("Upload a test image first") + url = f"http://s3:8333/{bucket}/{row['object_key']}" + req = urllib.request.Request(url) + try: + with urllib.request.urlopen(req, timeout=3) as response: + raise SystemExit(f"FAIL: anonymous S3 GET returned {response.status}") + except urllib.error.HTTPError as exc: + if exc.code != 403: + raise SystemExit(f"FAIL: expected 403, got {exc.code}") + print("PASS: anonymous S3 GET rejected with 403 (private bucket)") + finally: + pool.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/experiment_uploads.py b/scripts/experiment_uploads.py new file mode 100644 index 00000000..109f0942 --- /dev/null +++ b/scripts/experiment_uploads.py @@ -0,0 +1,71 @@ +"""10 simultaneous real uploads + Docker RSS observations (NO invented metrics).""" +import concurrent.futures +import json +import subprocess +import sys +import threading +import time +from pathlib import Path +from statistics import median +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check, arguments, run + + +def mib(value): + value = value.strip() + for suffix, multiplier in (("GiB",1024),("MiB",1),("KiB",1/1024),("B",1/1048576)): + if value.endswith(suffix): + return float(value[:-len(suffix)]) * multiplier + return None + + +def main(): + args = arguments('Concurrent uploads + observed API container memory') + client = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = client.bootstrap() + file = (ROOT / 'tests/fixtures/sample.png').read_bytes() + results, memory = [], [] + stop = threading.Event() + def observe(): + while not stop.is_set(): + r = subprocess.run(['docker','stats','--no-stream','--format','{{.Name}}|{{.MemUsage}}'], + capture_output=True, text=True, timeout=10) + if r.returncode == 0: + for line in r.stdout.splitlines(): + if '|api1-' in line or '|api2-' in line: + pass + parts = line.split('|') + if len(parts)==2 and ('api1-' in parts[0] or 'api2-' in parts[0]): + amount = mib(parts[1].split('/')[0]) + if amount is not None: + memory.append({'container': parts[0], 'mem_mib': round(amount,3)}) + stop.wait(0.2) + monitor = threading.Thread(target=observe, daemon=True) + monitor.start() + try: + def worker(i): + local = Client(args.base_url,args.ca_file,timeout=30) + start=time.perf_counter() + resp=local.call('POST', f'/api/v1/chats/{chat["id"]}/attachments?filename=run{i}.png', + 201, actor=alice, raw=file, headers={'Content-Type':'image/png'}, schema='Attachment') + return {'elapsed_ms':round((time.perf_counter()-start)*1000,2), 'instance': resp['headers'].get('X-Instance-Id')} + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + futures=[executor.submit(worker,i) for i in range(10)] + results=[f.result() for f in concurrent.futures.as_completed(futures)] + finally: + stop.set(); monitor.join(timeout=12) + output={'concurrency':10,'file_bytes':len(file),'completed':len(results), + 'median_latency_ms':round(median(x['elapsed_ms'] for x in results),2), + 'api_peak_mem_mib_observed':max([x['mem_mib'] for x in memory],default=None), + 'memory_samples':len(memory),'instances':sorted(set(x['instance'] for x in results))} + target=ROOT / 'evidence' / 'uploads-10.json' + target.write_text(json.dumps(output,ensure_ascii=False,indent=2),encoding='utf-8') + check(len(results)==10, 'not all uploads completed') + print('PASS: 10 parallel uploads; metrics written:',target) + print(json.dumps(output, ensure_ascii=False, indent=2)) + if not memory: + print('WARNING: no Docker memory samples; run docker stats separately for a valid grade-4 experiment') + +if __name__ == '__main__': + run(main) diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..2c2412ba --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,6 @@ +@echo off +cd /d "%~dp0.." +docker compose --env-file .env stop lb api1 api2 redis postgres s3 +if errorlevel 1 exit /b 1 +docker compose --env-file .env up -d --wait +exit /b %errorlevel% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..bf2873e9 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env stop lb api1 api2 redis postgres s3 +docker compose --env-file .env up -d --wait diff --git a/scripts/rolling-update.cmd b/scripts/rolling-update.cmd new file mode 100644 index 00000000..c1d13213 --- /dev/null +++ b/scripts/rolling-update.cmd @@ -0,0 +1,6 @@ +@echo off +cd /d "%~dp0.." +docker compose --env-file .env up -d --no-deps --force-recreate --wait api1 +if errorlevel 1 exit /b 1 +docker compose --env-file .env up -d --no-deps --force-recreate --wait api2 +exit /b %errorlevel% diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100644 index 00000000..f969fe61 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env up -d --no-deps --force-recreate --wait api1 +docker compose --env-file .env up -d --no-deps --force-recreate --wait api2 diff --git a/scripts/start-one.sh b/scripts/start-one.sh new file mode 100644 index 00000000..0a4f4b75 --- /dev/null +++ b/scripts/start-one.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +case "${1:-api1}" in + api1|api2) docker compose --env-file .env up -d "$1" ;; + *) echo "Usage: scripts/start-one.sh api1|api2" >&2; exit 2 ;; +esac diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..876261c5 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,6 @@ +@echo off +cd /d "%~dp0.." +if "%TARGET_INSTANCE_ID%"=="" set "TARGET_INSTANCE_ID=api1" +if not "%TARGET_INSTANCE_ID%"=="api1" if not "%TARGET_INSTANCE_ID%"=="api2" exit /b 2 +docker compose --env-file .env stop %TARGET_INSTANCE_ID% +exit /b %errorlevel% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..3ca4c1a0 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +case "${TARGET_INSTANCE_ID:-api1}" in + api1|api2) target="${TARGET_INSTANCE_ID:-api1}" ;; + *) echo "Unknown API replica" >&2; exit 2 ;; +esac +docker compose --env-file .env stop "$target" diff --git a/scripts/test_parallel.py b/scripts/test_parallel.py new file mode 100644 index 00000000..fd34a019 --- /dev/null +++ b/scripts/test_parallel.py @@ -0,0 +1,42 @@ +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import sys +import threading +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, check + + +def main(): + client = Client() + alice, _, _, chat = client.bootstrap() + key = uuid4().hex + path = "/api/v1/chats/" + chat["id"] + "/messages" + text = "Проверка одновременного повтора " + key + barrier = threading.Barrier(12) + + def send(_): + local = Client(timeout=10) + barrier.wait(timeout=10) + response = local.message(alice, chat, text, key=key) + return response["json"], response["headers"]["X-Instance-Id"] + + with ThreadPoolExecutor(max_workers=12) as workers: + results = list(workers.map(send, range(12))) + + ids = {message["id"] for message, _ in results} + instances = {instance for _, instance in results} + check(len(ids) == 1, "Повтор создал несколько сообщений") + check(len(instances) == 2, "Не удалось проверить обе реплики") + check(all(result[0] == results[0][0] for result in results), "Ответы различаются") + client.call("POST", path, 409, actor=alice, body={"text": "Другой текст"}, + headers={"Idempotency-Key": key}) + page = client.call("GET", path + "?limit=100", actor=alice, schema="MessagePage")["json"] + check(sum(item["id"] in ids for item in page["items"]) == 1, + "В истории найден дубль сообщения") + print("PASS: 12 параллельных запросов; 2 реплики; 1 сообщение; повтор с другим текстом — 409") + + +if __name__ == "__main__": + main() diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..fbb7bde3 --- /dev/null +++ b/src/db.py @@ -0,0 +1,29 @@ +import os + +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + + +def connection_kwargs(): + return { + "host": os.getenv("POSTGRES_HOST", "postgres"), + "port": int(os.getenv("POSTGRES_PORT", "5432")), + "dbname": os.environ["POSTGRES_DB"], + "user": os.environ["POSTGRES_USER"], + "password": os.environ["POSTGRES_PASSWORD"], + "connect_timeout": 5, + "options": "-c statement_timeout=5000 -c lock_timeout=3000", + "autocommit": True, + "row_factory": dict_row, + } + + +pool = ConnectionPool( + kwargs=connection_kwargs(), + min_size=1, + max_size=5, + max_waiting=20, + timeout=3, + check=ConnectionPool.check_connection, + open=False, +) diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..73e36a37 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,14 @@ +class ApiError(Exception): + def __init__(self, status, code, message): + super().__init__(message) + self.status = status + self.code = code + self.message = message + + +def invalid(message="Неверный запрос"): + raise ApiError(400, "invalid_request", message) + + +def unauthorized(): + raise ApiError(401, "unauthorized", "Неверные данные авторизации") diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..831595cc --- /dev/null +++ b/src/main.py @@ -0,0 +1,652 @@ +import base64 +import binascii +import hashlib +import json +import os +import logging +import re +import time +import tempfile +import threading +import unicodedata +from io import BytesIO +from contextlib import asynccontextmanager +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError +from fastapi import FastAPI, Header, Query, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response, StreamingResponse +from psycopg import InterfaceError, OperationalError +from psycopg.errors import ForeignKeyViolation, UniqueViolation +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from redis.exceptions import RedisError +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException + +from src.db import pool +from src.errors import ApiError, invalid, unauthorized +from src.pagination import encode_cursor, parse_page +from src.sessions import issue_session, redis_client +from src.storage import s3, bucket, check_storage, upload_original, download_original, delete_original, StorageFailure, MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, UnidentifiedImageError +Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS + + +log = logging.getLogger("messenger") +password_hasher = PasswordHasher() +INSTANCE_ID = os.getenv("INSTANCE_ID", uuid4().hex) +ready = False +IMAGE_DECODE_SLOT = threading.BoundedSemaphore(1) # prevent concurrent decoded-image RAM spikes + + +def parse_uuid(value): + try: + return UUID(value) + except (ValueError, TypeError, AttributeError): + invalid("Некорректный UUID") + + +def row_response(row): + result = {} + for key, value in row.items(): + if isinstance(value, UUID): + value = str(value) + elif isinstance(value, datetime): + value = value.astimezone(timezone.utc).isoformat( + timespec="microseconds" + ).replace("+00:00", "Z") + elif key in ("member_ids", "attachment_ids"): + value = [str(member) for member in value] + result[key] = value + return result + + +class InputModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + @field_validator("*", check_fields=False) + @classmethod + def reject_blank(cls, value): + if isinstance(value, str) and not value.strip(): + raise ValueError("Строка не должна быть пустой") + return value + + @field_validator("display_name", "title", "text", check_fields=False) + @classmethod + def reject_nul(cls, value): + if "\u0000" in value: + raise ValueError("Нулевой символ недопустим в текстовом поле") + return value + + +class CreateUser(InputModel): + username: str = Field(min_length=3, max_length=32) + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("username") + @classmethod + def validate_username(cls, value): + if re.fullmatch(r"[a-z0-9_]{3,32}", value) is None: + raise ValueError("Некорректный username") + return value + + +class CreateChat(InputModel): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + text: str = Field(min_length=0, max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @model_validator(mode="after") + def validate_message(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Нужен текст или вложение") + if "\x00" in self.text: + raise ValueError("Недопустимый текст") + if len(set(self.attachment_ids)) != len(self.attachment_ids): + raise ValueError("Повторные attachment_ids") + for value in self.attachment_ids: + try: + UUID(value) + except (ValueError, TypeError): + raise ValueError("Некорректный attachment_id") from None + return self + + +def bearer_token(value): + scheme, _, token = (value or "").partition(" ") + if scheme.lower() != "bearer" or not token: + unauthorized() + return token + + +def current_user(value): + token = bearer_token(value) + user_id = redis_client.get("session:" + token) + if user_id is None: + unauthorized() + with pool.connection() as conn: + user = conn.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (UUID(user_id),), + ).fetchone() + if user is None: + unauthorized() + return row_response(user) + + +@asynccontextmanager +async def lifespan(app): + global ready + try: + await run_in_threadpool(pool.open, wait=True, timeout=10) + await run_in_threadpool(redis_client.ping) + # Container may have started before the S3 service finished bootstrap. + storage_ok = False + for _ in range(30): + try: + await run_in_threadpool(check_storage) + storage_ok = True + break + except (BotoCoreError, ClientError, StorageFailure): + import asyncio + await asyncio.sleep(2) + if not storage_ok: + raise RuntimeError("S3 not ready") + ready = True + log.info("startup ready=true") + yield + finally: + ready = False + redis_client.close() + pool.close() + log.info("shutdown complete") + + +app = FastAPI(title="Lab 4 Messenger", lifespan=lifespan) + + +def error_response(request, status, code, message): + request_id = request.state.request_id + headers = {"X-Request-Id": request_id} + if status == 401: + if request.method == "POST" and request.url.path == "/api/v1/auth/sessions": + headers["WWW-Authenticate"] = 'Basic realm="messenger", charset="UTF-8"' + else: + headers["WWW-Authenticate"] = 'Bearer realm="messenger"' + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers=headers, + ) + + +@app.exception_handler(ApiError) +async def api_error(request, exc): + return error_response(request, exc.status, exc.code, exc.message) + + +@app.exception_handler(RequestValidationError) +async def validation_error(request, exc): + return error_response(request, 400, "invalid_request", "Некорректные поля или JSON") + + +@app.exception_handler((StorageFailure)) +async def storage_error(request, exc): + return error_response(request, 503, "unavailable", "Хранилище недоступно") + +@app.exception_handler(ClientError) +async def boto_error(request, exc): + return error_response(request, 503, "unavailable", "Хранилище недоступно") + +@app.exception_handler(StarletteHTTPException) +async def http_error(request, exc): + code = "not_found" if exc.status_code == 404 else "invalid_request" + return error_response(request, exc.status_code, code, "Некорректный запрос") + + +@app.middleware("http") +async def process_request(request: Request, call_next): + request.state.request_id = str(uuid4()) + started = time.perf_counter() + try: + is_login = request.method == "POST" and request.url.path == "/api/v1/auth/sessions" + public = request.method == "POST" and request.url.path in ( + "/api/v1/users", "/api/v1/auth/sessions" + ) + if request.url.path.startswith("/api/v1/") and not public: + request.state.user = await run_in_threadpool( + current_user, request.headers.get("Authorization") + ) + if is_login: + if await request.body(): + invalid("Для входа тело запроса должно быть пустым") + elif request.method == "POST" and request.url.path.startswith("/api/v1/"): + media_type = request.headers.get("content-type", "").split(";")[0].strip().lower() + if re.fullmatch(r"/api/v1/chats/[^/]+/attachments", request.url.path): + if media_type not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "Только PNG/JPEG") + elif media_type != "application/json": + raise ApiError(415, "unsupported_media_type", "Нужен application/json") + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc.status, exc.code, exc.message) + except (OperationalError, InterfaceError, RedisError, StorageFailure, BotoCoreError, ClientError): + response = error_response(request, 503, "unavailable", "Хранилище недоступно") + except Exception as exc: + log.error("request_failed type=%s request_id=%s", + type(exc).__name__, request.state.request_id) + response = error_response(request, 500, "internal_error", "Внутренняя ошибка") + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + route = request.scope.get("route") + log.info("request method=%s route=%s status=%s duration_ms=%.2f request_id=%s", + request.method, getattr(route, "path", "unmatched"), response.status_code, + (time.perf_counter() - started) * 1000, request.state.request_id) + return response + + +@app.get("/health/live") +def live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def readiness(): + if not ready: + raise ApiError(503, "unavailable", "Приложение не готово") + with pool.connection() as conn: + conn.execute("SELECT 1") + redis_client.ping() + check_storage() + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +def create_user(body: CreateUser): + password_hash = password_hasher.hash(body.password) + try: + with pool.connection() as conn: + user = conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) " + "VALUES (%s, %s, %s, %s) RETURNING id, username, display_name, created_at", + (uuid4(), body.username, body.display_name, password_hash), + ).fetchone() + except UniqueViolation as exc: + if exc.diag.constraint_name == "users_username_key": + raise ApiError(409, "conflict", "Username занят") from None + raise + return row_response(user) + + +@app.post("/api/v1/auth/sessions", status_code=201) +def login(authorization: str | None = Header(default=None)): + scheme, _, encoded = (authorization or "").partition(" ") + if scheme.lower() != "basic" or not encoded: + unauthorized() + try: + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, binascii.Error): + unauthorized() + if re.fullmatch(r"[a-z0-9_]{3,32}", username) is None or not 12 <= len(password) <= 128: + unauthorized() + with pool.connection() as conn: + user = conn.execute( + "SELECT id, username, display_name, password_hash, created_at " + "FROM users WHERE username = %s", (username,), + ).fetchone() + if user is None: + unauthorized() + try: + password_hasher.verify(user["password_hash"], password) + except VerifyMismatchError: + unauthorized() + token, expires_at = issue_session(user["id"]) + del user["password_hash"] + return {"token": token, "token_type": "Bearer", "expires_at": expires_at, + "user": row_response(user)} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def logout(request: Request): + token = bearer_token(request.headers.get("Authorization")) + redis_client.delete("session:" + token) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +def get_me(request: Request): + return request.state.user + + +def require_member(conn, chat_id, user_id): + found = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, UUID(user_id)), + ).fetchone() + if found is None: + raise ApiError(404, "not_found", "Чат не найден") + + +CHAT_SELECT = """ + SELECT c.id, c.title, c.created_at, + ARRAY(SELECT user_id FROM chat_members + WHERE chat_id = c.id ORDER BY user_id) AS member_ids + FROM chats c + JOIN chat_members mine ON mine.chat_id = c.id + WHERE mine.user_id = %s +""" + +MESSAGE_SELECT = """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT ma.attachment_id FROM message_attachments ma + WHERE ma.message_id=m.id ORDER BY ma.position)::uuid[] AS attachment_ids + FROM messages m +""" + +HISTORY_FIRST_SQL = MESSAGE_SELECT + """ + WHERE m.chat_id = %s ORDER BY m.created_at, m.id LIMIT %s +""" + +HISTORY_AFTER_SQL = MESSAGE_SELECT + """ + WHERE m.chat_id = %s AND (m.created_at, m.id) > (%s, %s) + ORDER BY m.created_at, m.id LIMIT %s +""" + + +def page_response(rows, size, scope): + items = [row_response(row) for row in rows[:size]] + cursor = encode_cursor(scope, items[-1]) if len(rows) > size else None + return {"items": items, "next_cursor": cursor} + + +@app.post("/api/v1/chats", status_code=201) +def create_chat(body: CreateChat, request: Request): + members = sorted(parse_uuid(value) for value in body.member_ids) + if len(members) != len(set(members)): + invalid("Участники повторяются") + if UUID(request.state.user["id"]) not in members: + invalid("Вызывающий должен входить в member_ids") + try: + with pool.connection() as conn: + with conn.transaction(): + chat = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) " + "RETURNING id, title, created_at", (uuid4(), body.title), + ).fetchone() + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat["id"], member) for member in members], + ) + except ForeignKeyViolation: + invalid("Неизвестный участник") + chat["member_ids"] = members + return row_response(chat) + + +@app.get("/api/v1/chats") +def list_chats(request: Request, limit: str = Query(default="50"), + cursor: str | None = Query(default=None)): + user_id = request.state.user["id"] + scope = "chats:" + user_id + size, after = parse_page(limit, cursor, scope) + query = CHAT_SELECT + params = [UUID(user_id)] + if after is not None: + query += " AND (c.created_at, c.id) > (%s, %s)" + params.extend(after) + query += " ORDER BY c.created_at, c.id LIMIT %s" + params.append(size + 1) + with pool.connection() as conn: + rows = conn.execute(query, params).fetchall() + return page_response(rows, size, scope) + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, request: Request): + with pool.connection() as conn: + chat = conn.execute( + CHAT_SELECT + " AND c.id = %s", + (UUID(request.state.user["id"]), parse_uuid(chat_id)), + ).fetchone() + if chat is None: + raise ApiError(404, "not_found", "Чат не найден") + return row_response(chat) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message( + chat_id: str, + body: CreateMessage, + request: Request, + idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"), +): + if idempotency_key is None or re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key) is None: + invalid("Некорректный Idempotency-Key") + + chat_uuid = parse_uuid(chat_id) + user_id = UUID(request.state.user["id"]) + digest = hashlib.sha256( + json.dumps({"text": body.text, "attachment_ids": body.attachment_ids}, ensure_ascii=False, sort_keys=True).encode("utf-8") + ).hexdigest() + + with pool.connection() as conn: + with conn.transaction(): + require_member(conn, chat_uuid, str(user_id)) + if body.attachment_ids: + attachment_uuids = [UUID(v) for v in body.attachment_ids] + found = conn.execute( + "SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s)", + (attachment_uuids,), + ).fetchall() + if len(found) != len(attachment_uuids) or any(row["chat_id"] != chat_uuid for row in found): + raise ApiError(404, "not_found", "Вложение не найдено") + if any(row["status"] != "ready" for row in found): + raise ApiError(409, "conflict", "Вложение не готово") + conn.execute( + "DELETE FROM message_requests WHERE sender_id = %s AND chat_id = %s " + "AND request_key = %s AND created_at < now() - interval '24 hours'", + (user_id, chat_uuid, idempotency_key), + ) + created = conn.execute( + "INSERT INTO message_requests (sender_id, chat_id, request_key, request_hash) " + "VALUES (%s, %s, %s, %s) ON CONFLICT DO NOTHING RETURNING request_key", + (user_id, chat_uuid, idempotency_key, digest), + ).fetchone() + if created is None: + existing = conn.execute( + "SELECT request_hash, message_id FROM message_requests " + "WHERE sender_id = %s AND chat_id = %s AND request_key = %s", + (user_id, chat_uuid, idempotency_key), + ).fetchone() + if existing is None: + raise ApiError(503, "unavailable", "Повторите запрос") + if existing["request_hash"] != digest: + raise ApiError(409, "conflict", "Ключ уже использован для другого сообщения") + message = conn.execute( + MESSAGE_SELECT + " WHERE m.id = %s", + (existing["message_id"],), + ).fetchone() + else: + message = conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text) " + "VALUES (%s, %s, %s, %s) " + "RETURNING id, chat_id, sender_id, text, created_at", + (uuid4(), chat_uuid, user_id, body.text), + ).fetchone() + for position, attach in enumerate(body.attachment_ids): + conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)", + (message["id"], UUID(attach), position), + ) + conn.execute( + "UPDATE message_requests SET message_id = %s " + "WHERE sender_id = %s AND chat_id = %s AND request_key = %s", + (message["id"], user_id, chat_uuid, idempotency_key), + ) + if "attachment_ids" not in message: + message["attachment_ids"] = [UUID(v) for v in body.attachment_ids] + return row_response(message) + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: str, request: Request, limit: str = Query(default="50"), + cursor: str | None = Query(default=None)): + chat_uuid = parse_uuid(chat_id) + scope = "messages:" + str(chat_uuid) + size, after = parse_page(limit, cursor, scope) + with pool.connection() as conn: + require_member(conn, chat_uuid, request.state.user["id"]) + if after is None: + rows = conn.execute(HISTORY_FIRST_SQL, (chat_uuid, size + 1)).fetchall() + else: + rows = conn.execute( + HISTORY_AFTER_SQL, (chat_uuid, after[0], after[1], size + 1) + ).fetchall() + return page_response(rows, size, scope) + + +# Lab 4: binary media are kept in private S3; PostgreSQL stores only metadata. +# Raw request bodies are streamed in bounded chunks to a temporary spool file. +def validate_image(stream, advertised_type): + stream.seek(0) + try: + with IMAGE_DECODE_SLOT, Image.open(stream) as probe: + actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(probe.format) + width, height = probe.size + if width * height > MAX_IMAGE_PIXELS: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + if actual != advertised_type: + raise ApiError(415, "unsupported_media_type", "Тип изображения не совпадает с содержимым") + probe.verify() + stream.seek(0) + # Verifying full decode rejects truncated/corrupt image streams. + with IMAGE_DECODE_SLOT, Image.open(stream) as image: + image.load() + except ApiError: + raise + except (UnidentifiedImageError, OSError, ValueError, SyntaxError, Image.DecompressionBombError): + raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from None + stream.seek(0) + + +def attachment_response(row): + result = row_response(row) + result.pop("object_key", None) + result["variants"] = [] + result["error_code"] = None + return result + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def create_attachment(chat_id: str, request: Request, filename: str = Query(...)): + chat_uuid = parse_uuid(chat_id) + user_id = UUID(request.state.user["id"]) + if not 1 <= len(filename) <= 255 or any(c in filename for c in "/\\\x00") or any(unicodedata.category(c) == "Cc" for c in filename): + invalid("Опасное имя файла") + content_type = request.headers.get("content-type", "").split(";")[0].strip().lower() + length_header = request.headers.get("content-length") + if length_header and length_header.isdigit() and int(length_header) > MAX_UPLOAD_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 MiB") + # ACL before touching storage. + await run_in_threadpool(check_membership, chat_uuid, user_id) + file = tempfile.SpooledTemporaryFile(max_size=256 * 1024) + digest = hashlib.sha256() + size = 0 + try: + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_UPLOAD_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 MiB") + digest.update(chunk) + file.write(chunk) + if size == 0: + raise ApiError(415, "unsupported_media_type", "Пустой файл") + await run_in_threadpool(validate_image, file, content_type) + attachment_id = uuid4() + object_key = f"originals/{attachment_id.hex}" + file.seek(0) + # One S3 PUT; no multipart uploads left by a client disconnect. + try: + await run_in_threadpool(upload_original, file, object_key, size, content_type) + except (StorageFailure, BotoCoreError, ClientError): + raise ApiError(503, "unavailable", "Объектное хранилище недоступно") from None + try: + result = await run_in_threadpool(save_attachment, attachment_id, chat_uuid, user_id, filename, + content_type, size, digest.hexdigest(), object_key) + except Exception: + # Best effort compensation; orphan reconciler handles crash between S3 PUT and DB COMMIT. + try: + await run_in_threadpool(delete_original, object_key) + except Exception: + log.warning("upload compensation failed for %s", object_key) + raise + return attachment_response(result) + finally: + file.close() + + +def check_membership(chat_uuid, user_id): + with pool.connection() as conn: + require_member(conn, chat_uuid, str(user_id)) + + +def save_attachment(attachment_id, chat_uuid, user_id, filename, content_type, size, digest, key): + with pool.connection() as conn: + with conn.transaction(): + require_member(conn, chat_uuid, str(user_id)) + return conn.execute( + "INSERT INTO attachments (id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,object_key) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,'ready',%s) RETURNING *", + (attachment_id, chat_uuid, user_id, filename, content_type, size, digest, key) + ).fetchone() + + +def owned_attachment(attachment_id, user_id): + with pool.connection() as conn: + row = conn.execute( + "SELECT a.* FROM attachments a JOIN chat_members cm ON cm.chat_id = a.chat_id " + "WHERE a.id = %s AND cm.user_id = %s", + (attachment_id, UUID(user_id)), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Вложение не найдено") + return row + + +@app.get("/api/v1/attachments/{attachment_id}") +def attachment_metadata(attachment_id: str, request: Request): + return attachment_response(owned_attachment(parse_uuid(attachment_id), request.state.user["id"])) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def attachment_content(attachment_id: str, request: Request, variant: str = Query(default="original")): + row = owned_attachment(parse_uuid(attachment_id), request.state.user["id"]) + if variant != "original": + raise ApiError(404, "not_found", "Вариант не найден") + if row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение не готово") + try: + s3_body = download_original(row["object_key"]) + except (BotoCoreError, ClientError, StorageFailure): + raise ApiError(503, "unavailable", "Объектное хранилище недоступно") from None + def chunks(): + try: + for data in s3_body.iter_chunks(chunk_size=64 * 1024): + if data: + yield data + finally: + s3_body.close() + return StreamingResponse(chunks(), media_type=row["content_type"], + headers={"Content-Length": str(row["size_bytes"]), "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff"}) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..25bd7d36 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,36 @@ +from pathlib import Path + +import psycopg + +from src.db import connection_kwargs + + +def main(): + files = sorted((Path(__file__).resolve().parents[1] / "migrations").glob("*.sql")) + if not files: + raise RuntimeError("Миграции не найдены") + with psycopg.connect(**connection_kwargs()) as conn: + conn.execute(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """) + for file in files: + applied = False + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(202602)") + exists = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", (file.name,) + ).fetchone() + if exists is None: + conn.execute(file.read_text(encoding="utf-8")) + conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", (file.name,) + ) + applied = True + print(("APPLIED " if applied else "SKIP ") + file.name) + + +if __name__ == "__main__": + main() diff --git a/src/pagination.py b/src/pagination.py new file mode 100644 index 00000000..03615854 --- /dev/null +++ b/src/pagination.py @@ -0,0 +1,47 @@ +import base64 +import binascii +import json +import re +from datetime import datetime +from uuid import UUID + +from src.errors import invalid + + +def encode_cursor(scope, item): + data = {"scope": scope, "created_at": item["created_at"], "id": item["id"]} + raw = json.dumps(data, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def parse_page(limit, cursor, scope): + if len(limit) > 10 or re.fullmatch(r"[0-9]+", limit) is None: + invalid("limit должен быть целым числом от 1 до 100") + size = int(limit) + if not 1 <= size <= 100: + invalid("limit должен быть от 1 до 100") + if cursor is None: + return size, None + if not cursor or len(cursor) > 2048: + invalid("Некорректный cursor") + try: + raw = base64.b64decode( + cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True + ) + data = json.loads(raw) + if not isinstance(data, dict) or set(data) != {"scope", "created_at", "id"}: + raise ValueError + if data["scope"] != scope or not isinstance(data["created_at"], str): + raise ValueError + if re.fullmatch( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?Z", + data["created_at"], + ) is None: + raise ValueError + created_at = datetime.fromisoformat(data["created_at"][:-1] + "+00:00") + if created_at.tzinfo is None: + raise ValueError + item_id = UUID(data["id"]) + except (ValueError, TypeError, AttributeError, binascii.Error): + invalid("Некорректный cursor или cursor от другого ресурса") + return size, (created_at, item_id) diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..afd3c4d0 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,57 @@ +"""Dry-run by default: find unused S3 originals older than safe grace period. + +Usage: + docker compose exec -T api1 python -m src.reconcile + docker compose exec -T api1 python -m src.reconcile --apply --min-age-seconds 3600 + +No database objects or live originals are ever deleted. Only orphans, and only +under the server-owned originals/ prefix, are eligible for cleanup. +""" +import argparse +import datetime as dt +import json +import logging +from src.db import connection_kwargs +from src.storage import s3, bucket +import psycopg + +log = logging.getLogger("attachment-reconcile") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--apply", action="store_true", help="Delete eligible orphan objects") + parser.add_argument("--min-age-seconds", type=int, default=3600) + args = parser.parse_args() + if args.min_age_seconds < 3600: + parser.error("Safety grace period must be at least 3600s") + now = dt.datetime.now(dt.timezone.utc) + checked = 0 + orphan = 0 + removed = 0 + with psycopg.connect(**connection_kwargs()) as conn: + for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix="originals/"): + for item in page.get("Contents", []): + checked += 1 + key = item["Key"] + age = (now - item["LastModified"]).total_seconds() + if age < args.min_age_seconds: + continue + exists = conn.execute("SELECT 1 FROM attachments WHERE object_key=%s", (key,)).fetchone() + if exists: + continue + orphan += 1 + if args.apply: + # Recheck immediately before deletion; do not delete newly referenced keys. + exists = conn.execute("SELECT 1 FROM attachments WHERE object_key=%s", (key,)).fetchone() + if exists: + continue + s3.delete_object(Bucket=bucket, Key=key) + removed += 1 + print(json.dumps({"mode": "apply" if args.apply else "dry-run", "scanned": checked, + "eligible_orphans": orphan, "removed": removed, + "min_age_seconds": args.min_age_seconds}, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/src/run.py b/src/run.py new file mode 100644 index 00000000..d01a0b04 --- /dev/null +++ b/src/run.py @@ -0,0 +1,23 @@ +import logging +import os + +import uvicorn + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + logging.getLogger("messenger").info( + "config app_env=%s", os.getenv("APP_ENV", "lab") + ) + uvicorn.run( + "src.main:app", + host=os.getenv("APP_HOST", "0.0.0.0"), + port=int(os.getenv("APP_PORT", "8080")), + workers=1, + timeout_graceful_shutdown=5, + timeout_keep_alive=1, + access_log=False, + ) \ No newline at end of file diff --git a/src/sessions.py b/src/sessions.py new file mode 100644 index 00000000..82854049 --- /dev/null +++ b/src/sessions.py @@ -0,0 +1,33 @@ +import os +import secrets +import time +from datetime import datetime, timezone + +from redis import Redis +from redis.backoff import NoBackoff +from redis.retry import Retry + + +redis_client = Redis( + host=os.getenv("REDIS_HOST", "redis"), + port=int(os.getenv("REDIS_PORT", "6379")), + decode_responses=True, + socket_connect_timeout=2, + socket_timeout=2, + max_connections=10, + retry=Retry(NoBackoff(), 0), +) + +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +if SESSION_TTL_SECONDS <= 0: + raise RuntimeError("SESSION_TTL_SECONDS должен быть положительным") + + +def issue_session(user_id): + token = secrets.token_urlsafe(32) + expires_ms = int(time.time() * 1000) + SESSION_TTL_SECONDS * 1000 + redis_client.set("session:" + token, str(user_id), pxat=expires_ms) + expires_at = datetime.fromtimestamp( + expires_ms / 1000, timezone.utc + ).isoformat(timespec="milliseconds").replace("+00:00", "Z") + return token, expires_at diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..45ba7d89 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,43 @@ +"""S3-compatible private storage. Only API-scoped credentials are available here.""" +import os +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError, BotoCoreError + +MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) +bucket = os.getenv("S3_BUCKET", "messenger-attachments") +s3 = boto3.client( + "s3", endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"), + region_name=os.getenv("S3_REGION", "us-east-1"), + aws_access_key_id=os.environ.get("S3_ACCESS_KEY_ID", "unset"), + aws_secret_access_key=os.environ.get("S3_SECRET_ACCESS_KEY", "unset"), + config=Config(s3={"addressing_style": "path"}, connect_timeout=2, read_timeout=30, + retries={"mode": "standard", "max_attempts": 2}), +) + +class StorageFailure(Exception): + pass + + +def check_storage(): + # Scoped user has ListBucket but no privileges outside this bucket. + s3.list_objects_v2(Bucket=bucket, MaxKeys=1) + + +def upload_original(file, key, size, content_type): + file.seek(0) + s3.put_object(Bucket=bucket, Key=key, Body=file, ContentLength=size, ContentType=content_type) + + +def delete_original(key): + s3.delete_object(Bucket=bucket, Key=key) + + +def download_original(key): + try: + return s3.get_object(Bucket=bucket, Key=key)["Body"] + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + raise StorageFailure("Object missing") from None + raise diff --git a/student-tests/test_images.py b/student-tests/test_images.py new file mode 100644 index 00000000..e0de17e0 --- /dev/null +++ b/student-tests/test_images.py @@ -0,0 +1,33 @@ +"""Unit-level image checks; run in built api1 container.""" +import tempfile +import unittest +from pathlib import Path +from src.main import validate_image +from src.errors import ApiError + +PNG = (Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes() + + +class ImageValidationTest(unittest.TestCase): + def check(self, content, content_type): + with tempfile.SpooledTemporaryFile() as stream: + stream.write(content) + validate_image(stream, content_type) + + def test_fixture(self): + self.check(PNG,'image/png') + + def test_corrupt(self): + with self.assertRaises(ApiError): + self.check(PNG[:70], 'image/png') + + def test_spoof_mime(self): + with self.assertRaises(ApiError): + self.check(PNG,'image/jpeg') + + def test_random_bytes(self): + with self.assertRaises(ApiError): + self.check(b'unrelated bytes', 'image/png') + +if __name__=='__main__': + unittest.main()