diff --git a/.env.example b/.env.example index d99d26a2..d2f1aef5 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,17 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 +# Copy to .env and choose local credentials. Never commit .env. APP_PORT=8080 -APP_ENV=lab +POSTGRES_DB=messenger +POSTGRES_USER=messenger +POSTGRES_PASSWORD=replace-with-local-password SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +CACHE_TTL_SECONDS=30 +MAX_REQUEST_BYTES=65536 +MAX_INFLIGHT=32 +MINIO_ROOT_USER=local-admin +MINIO_ROOT_PASSWORD=replace-with-admin-password +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=replace-with-api-password MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Compose sets internal PostgreSQL, Redis and S3 addresses. +# S3 bucket: messenger-attachments. Root credentials go only to s3/s3-init. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..37fc8fda 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: не указаны +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: не заявлен -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: перенесены PostgreSQL, Redis, Nginx и две API-реплики из lab3; добавлены private MinIO bucket, Attachment, upload/download через API, миграция, очистка orphan-объектов и проверки вложений. +Команды запуска из чистого clone: `cp .env.example .env`, задать локальные пароли, затем `docker compose --env-file .env up -d --build` и `make test`. +Результаты проверок и ссылка на REPORT.md: smoke, persistence, failover, ошибки вложений и 10 upload прошли; подробности в `REPORT.md`. +Ограничения / известные ошибки: S3, PostgreSQL и Redis по одному экземпляру; cleanup удаляет только orphan-объекты старше одного часа. +Источники и использование ИИ: FastAPI, SQLAlchemy, boto3, Pillow, MinIO; Codex использовался для реализации и проверок. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..1616ebd0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12.8-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY contracts ./contracts +COPY src ./src +COPY migrations ./migrations +COPY tests ./tests +COPY student-tests ./student-tests +COPY scripts ./scripts +COPY lab.json . + +RUN useradd --create-home --shell /usr/sbin/nologin appuser +USER appuser + +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()" + +STOPSIGNAL SIGTERM +CMD ["python", "-m", "src"] diff --git a/Makefile b/Makefile index 5d86d171..fe46f96d 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load upload-load test-persistence test-failover help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -21,6 +21,9 @@ test: load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) +upload-load: + $(PYTHON) scripts/upload_load.py --base-url "$(BASE_URL)" + test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..64f0952c 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,41 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: не указаны +- Целевая оценка: 4 +- Commit/PR: не опубликованы +- Использованные библиотеки и помощь ИИ: FastAPI, SQLAlchemy, Redis, boto3, Pillow, MinIO; Codex использовался для реализации и проверок. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +```sh +cp .env.example .env +# Задать свои POSTGRES_PASSWORD, MINIO_ROOT_PASSWORD и S3_SECRET_ACCESS_KEY. +docker compose --env-file .env up -d --build +make test +python student-tests/test_attachments.py +python scripts/upload_load.py --concurrency 10 --output evidence/upload-load.json +``` + +Стенд запускает PostgreSQL, Redis, MinIO, миграции, две API-реплики и Nginx. Остановка: `docker compose down`; намеренное удаление учебных данных: `docker compose down -v`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Из lab3 перенесены PostgreSQL, Redis-сессии, Nginx и две API-реплики. Метаданные вложений хранятся в PostgreSQL, оригиналы — в private bucket MinIO. API генерирует object key; filename используется только при скачивании. + +Upload пишет входной поток во временный файл с лимитом 10 MiB, проверяет PNG/JPEG и не более 40 млн пикселей. Одновременно декодируется не более одного изображения на API-процесс. Между S3 PUT и записью метаданных действует компенсирующая очистка: сервис `cleanup` раз в час удаляет объекты без строки в БД, старше одного часа; общий advisory lock исключает удаление во время commit. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. - -## Эксперименты и отказы - -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Публичный smoke | `make test` | PASS: lab4 public API scenario | console | +| Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | прежняя сессия, сообщение и оригинал сохранены | console | +| Failover | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` | PASS; восстановление записи за 6.361 с, файл читается через другую API-реплику | console | +| Ошибки вложений | `python student-tests/test_attachments.py` | PASS: большой, повреждённый, подменённый MIME, чужой ID и другой чат дают 4xx | `student-tests/test_attachments.py` | +| 10 upload | `python scripts/upload_load.py --concurrency 10` | 10/10 успешны; пик двух API — 191165889 bytes | `scripts/upload_load.py` | +| Private bucket | `docker compose exec -T nginx wget -S -O /dev/null http://s3:9000/messenger-attachments` | unauthenticated GET возвращает 403 | console | +| S3 unavailable | pause S3, `GET /health/ready`, unpause S3 | через LB: 503; после восстановления: 200 | console | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +MinIO, PostgreSQL и Redis запущены по одному экземпляру. Cleanup предназначен для orphan-объектов; он не восстанавливает отсутствующие объекты для существующих метаданных. Административная консоль MinIO не публикуется в сеть. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..60daad12 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,175 @@ +x-api: &api + image: mtusi2026lab4-api:local + build: . + dns_opt: + - timeout:1 + - attempts:1 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + POSTGRES_HOST: postgres + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-30} + MAX_REQUEST_BYTES: ${MAX_REQUEST_BYTES:-65536} + MAX_INFLIGHT: ${MAX_INFLIGHT:-32} + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + networks: [backend] + cpus: 1.0 + mem_limit: 512m + stop_grace_period: 10s + +services: + api: + <<: *api + deploy: + replicas: 2 + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + s3-init: + condition: service_completed_successfully + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1).read()"] + interval: 2s + timeout: 2s + start_period: 5s + retries: 2 + + migrate: + <<: *api + command: [python, -m, src.database] + healthcheck: + disable: true + restart: 'no' + depends_on: + postgres: + condition: service_healthy + + s3: + # RELEASE.2025-09-07T16-13-09Z, pinned by immutable image digest. + image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: [server, /data, --console-address, '127.0.0.1:9001'] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env} + MINIO_BROWSER: 'off' + volumes: + - s3_data:/data + networks: [backend] + healthcheck: + test: [CMD, curl, -f, http://127.0.0.1:9000/minio/health/live] + interval: 2s + timeout: 2s + retries: 30 + cpus: 1.0 + mem_limit: 512m + + s3-init: + # The pinned server image also contains mc; no second image is needed. + image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + entrypoint: [sh, /bootstrap.sh] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + volumes: + - ./infra/s3-bootstrap.sh:/bootstrap.sh:ro + - ./infra/s3-policy.json:/policy.json:ro + depends_on: + s3: + condition: service_healthy + networks: [backend] + restart: 'no' + cpus: 0.25 + mem_limit: 128m + + cleanup: + <<: *api + command: [python, -m, src.cleanup] + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + restart: unless-stopped + cpus: 0.25 + mem_limit: 128m + + nginx: + image: nginx:1.28.0-alpine + entrypoint: [sh, /etc/nginx/readiness.sh] + dns_opt: + - timeout:1 + - attempts:1 + ports: + - '127.0.0.1:${APP_PORT:-8080}:8080' + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/readiness.sh:/etc/nginx/readiness.sh:ro + networks: [edge, backend] + depends_on: + api: + condition: service_healthy + healthcheck: + test: [CMD-SHELL, 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready'] + interval: 2s + timeout: 2s + retries: 3 + cpus: 0.5 + mem_limit: 128m + stop_grace_period: 10s + + postgres: + image: postgres:17.4-bookworm + environment: + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: + - postgres_data:/var/lib/postgresql/data + networks: [backend] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 2s + timeout: 2s + retries: 30 + cpus: 1.0 + mem_limit: 512m + + redis: + image: redis:7.4.2-bookworm + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory", "128mb", "--maxmemory-policy", "noeviction"] + volumes: + - redis_data:/data + networks: [backend] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 30 + cpus: 0.5 + mem_limit: 256m + +networks: + edge: + backend: + internal: true +volumes: + postgres_data: + redis_data: + s3_data: diff --git a/infra/baseline.md b/infra/baseline.md new file mode 100644 index 00000000..a70c26b5 --- /dev/null +++ b/infra/baseline.md @@ -0,0 +1,34 @@ +# Перенесённая основа lab3 + +На этом этапе перенесена только работающая основа: FastAPI, PostgreSQL, +миграции 001–003, сессии и кеш Redis, идемпотентность сообщений, две API-реплики +и Nginx с проверкой готовности. S3 и вложения ещё не реализованы. +Контракт и публичные тесты остаются от lab4; полный smoke lab4 пока не пройдёт. + +## Запуск основы + +Скопируйте `.env.example` в `.env` и задайте свой `POSTGRES_PASSWORD`. +Параметры `*_FILE` и S3 пока являются настройками будущей реализации; +текущая основа использует `POSTGRES_*` и Redis из Compose. + +```sh +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +``` + +API доступен через http://localhost:8080. При одновременно запущенной lab3 +укажите другой `APP_PORT` в `.env`. Имя образа lab4 отдельное, volumes +изолируются именем Compose-проекта. Миграции выполняет сервис `migrate`. + +Проверка перенесённой логики после запуска: + +```sh +docker compose exec -T api python -m unittest discover -s student-tests -v +``` + +В `student-tests` перенесены проверки базового API/сессий и идемпотентности. +Эксперименты масштабирования, старые результаты и отчёт lab3 не переносились. +Скрипты `scripts/restart.*` и `scripts/stop-one.*` работают с текущим проектом; +для остановки реплики требуется `TARGET_INSTANCE_ID` из `X-Instance-Id`. +При добавлении S3 потребуется включить его в сценарий перезапуска и адаптировать +лимиты тела запроса в API и Nginx для загрузки изображений. diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 00000000..1cc308ba --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,38 @@ +worker_processes 1; +worker_shutdown_timeout 8s; +events { worker_connections 1024; } +http { + upstream messenger { + zone messenger 64k; + include /etc/nginx/ready-backends.conf; + } + log_format upstream '$request_method $uri $status upstream=$upstream_addr ' + 'upstream_status=$upstream_status time=$request_time'; + access_log /var/log/nginx/access.log upstream; + server { + listen 8080; + # API enforces the byte limit and returns the contracted JSON error. + client_max_body_size 0; + client_body_timeout 5s; + client_header_timeout 5s; + keepalive_timeout 5s; + location / { + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_request_buffering off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header Connection close; + proxy_connect_timeout 1s; + proxy_send_timeout 35s; + proxy_read_timeout 120s; + # A 503 is an API response (for example, S3 is unavailable), so + # preserve it for the client instead of marking every replica down. + proxy_next_upstream error timeout http_502 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 8s; + # Never transparently retry a POST already sent to an API. + # A client retries messages using the original Idempotency-Key. + } + } +} diff --git a/infra/readiness.sh b/infra/readiness.sh new file mode 100644 index 00000000..9a2e627f --- /dev/null +++ b/infra/readiness.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# Only ready API addresses enter the upstream, including at nginx startup. +update_backends() { + next=/etc/nginx/ready-backends.conf.next + : > "$next" + for address in $(getent ahostsv4 api | awk '{print $1}' | sort -u); do + if wget -q -T 1 -O /dev/null "http://$address:8080/health/ready"; then + echo "server $address:8080;" >> "$next" + fi + done + # If every running API temporarily loses a shared dependency (for example + # S3), retain the last addresses. The API can then return the contracted + # 503 instead of Nginx replacing it with a 502. At initial startup there + # is no previous healthy set, so use an inert placeholder. + if [ ! -s "$next" ]; then + if [ -s /etc/nginx/ready-backends.conf ] && ! grep -q '127.0.0.1:9 down' /etc/nginx/ready-backends.conf; then + rm -f "$next" + return + fi + echo 'server 127.0.0.1:9 down;' > "$next" + fi + if ! cmp -s "$next" /etc/nginx/ready-backends.conf; then + mv "$next" /etc/nginx/ready-backends.conf + if [ "${1:-}" = reload ] && [ -f /var/run/nginx.pid ]; then + nginx -s reload + fi + else + rm -f "$next" + fi +} + +update_backends +( + while sleep 2; do + update_backends reload + done +) & +exec nginx -g 'daemon off;' diff --git a/infra/s3-bootstrap.sh b/infra/s3-bootstrap.sh new file mode 100644 index 00000000..c0baf627 --- /dev/null +++ b/infra/s3-bootstrap.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +test "$MINIO_ROOT_USER" != "$S3_ACCESS_KEY_ID" +mc alias set local http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null +mc mb --ignore-existing local/messenger-attachments +mc anonymous set none local/messenger-attachments +mc admin user add local "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null +mc admin policy info local attachments >/dev/null 2>&1 || mc admin policy create local attachments /policy.json +mc admin policy attach local attachments --user "$S3_ACCESS_KEY_ID" diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 00000000..41bcff8c --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/originals/*"] + } + ] +} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..aee31d8f --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,30 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id), + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); +CREATE INDEX chats_order_idx ON chats (created_at, id); diff --git a/migrations/002_history_version.sql b/migrations/002_history_version.sql new file mode 100644 index 00000000..7b7972c1 --- /dev/null +++ b/migrations/002_history_version.sql @@ -0,0 +1 @@ +ALTER TABLE chats ADD COLUMN history_version bigint NOT NULL DEFAULT 0; diff --git a/migrations/003_message_idempotency.sql b/migrations/003_message_idempotency.sql new file mode 100644 index 00000000..434b1e28 --- /dev/null +++ b/migrations/003_message_idempotency.sql @@ -0,0 +1,9 @@ +-- Additive migration: no rewrite of existing lab2 tables or messages. +CREATE TABLE message_idempotency ( + actor_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (actor_id, chat_id, key) +); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..77317cf9 --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,18 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id), + object_key text NOT NULL UNIQUE, + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX attachments_chat_idx ON attachments(chat_id); + +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CHECK (char_length(text) <= 4000); +ALTER TABLE messages ADD CHECK (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0); +ALTER TABLE messages ADD CHECK (cardinality(attachment_ids) <= 10); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..86eff95c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,19 @@ +fastapi==0.115.6 +uvicorn==0.34.0 +pydantic==2.13.5 +starlette==0.41.3 +annotated-types==0.8.0 +anyio==4.15.1 +click==8.5.0 +colorama==0.4.6; sys_platform == "win32" +h11==0.16.0 +idna==3.20 +pydantic_core==2.46.5 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +psycopg[binary]==3.2.6 +SQLAlchemy==2.0.41 +redis==5.2.1 +argon2-cffi==23.1.0 +boto3==1.40.70 +Pillow==12.0.0 diff --git a/scripts/ops.py b/scripts/ops.py new file mode 100644 index 00000000..40567c41 --- /dev/null +++ b/scripts/ops.py @@ -0,0 +1,79 @@ +"""Project-scoped Docker operations; never read/export container environments.""" +import json +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def docker(*args): + return subprocess.check_output(['docker', *args], cwd=ROOT, text=True, timeout=120).strip() + + +def compose(*args): + return docker('compose', *args) + + +def snapshot(service=None): + ids = compose('ps', '-a', '-q', *([service] if service else [])).split() + if not ids: + return [] + # Select fields before returning; no secrets in evidence. + template = ('{"id":{{json .Id}},"name":{{json .Name}},"instance":{{json .Config.Hostname}},' + '"image":{{json .Image}},"state":{{json .State.Status}},' + '"started":{{json .State.StartedAt}},' + '"health":{{with index .State "Health"}}{{json .Status}}{{else}}null{{end}},' + '"cpu":{{.HostConfig.NanoCpus}},"memory":{{.HostConfig.Memory}},' + '"ports":{{json .HostConfig.PortBindings}}}') + return [json.loads(line) for line in docker('inspect', '--format', template, *ids).splitlines()] + + +def wait_api(count=2, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + rows = snapshot('api') + if len(rows) == count and all(r['health'] == 'healthy' and r['state'] == 'running' for r in rows): + time.sleep(3) # Allow the nginx readiness poll (2s) and reload. + return rows + time.sleep(0.5) + raise RuntimeError('API replicas did not become healthy') + + +def save(name, value): + (ROOT / 'evidence').mkdir(exist_ok=True) + (ROOT / 'evidence' / name).write_text(json.dumps(value, indent=2) + '\n', encoding='utf-8') + + +def stop_instance(instance): + before = snapshot('api') + matches = [r for r in before if r['instance'] == instance and r['state'] == 'running'] + if len(matches) != 1 or sum(r['state'] == 'running' for r in before) < 2: + raise RuntimeError('Target must identify one running API with a running peer') + docker('stop', '--time', '8', matches[0]['id']) + result = {'target': instance, 'before': before, 'after': snapshot('api')} + if next(r for r in result['after'] if r['instance'] == instance)['state'] != 'exited': + raise RuntimeError('Target was not stopped') + return result + + +def restore(): + compose('up', '-d', '--no-deps', '--no-recreate', 'api') + return wait_api() + + +def rolling(): + before = wait_api() + if len(before) != 2: + raise RuntimeError('This rolling scenario requires exactly two replicas') + steps = [] + for old in before: + stop_instance(old['instance']) + # Only remove the selected, already stopped API; never remove a volume. + docker('rm', old['id']) + compose('up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=2', 'api') + steps.append({'replaced': old['id'], 'ready': wait_api()}) + result = {'before': before, 'steps': steps, 'after': snapshot('api')} + if {r['id'] for r in before} & {r['id'] for r in result['after']}: + raise RuntimeError('Not all original API containers were replaced') + return result diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..95d967db --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +python scripts\restart.py diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 00000000..14932b1f --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,28 @@ +"""Restart the whole lab stack, retaining volumes and recording actual start times.""" +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def snapshot(): + ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], cwd=ROOT, text=True).split() + return subprocess.check_output( + ['docker', 'inspect', '--format', '{{.Name}} {{.State.StartedAt}}', *ids], + cwd=ROOT, text=True, + ) + + +def main(): + before = snapshot() + subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis', 's3', 'cleanup', 'nginx'], + cwd=ROOT, check=True, timeout=60) + after = snapshot() + (ROOT / 'evidence').mkdir(exist_ok=True) + (ROOT / 'evidence' / 'restart.txt').write_text( + 'Command: docker compose restart api postgres redis s3 cleanup nginx\nBefore (UTC):\n' + before + + 'After (UTC):\n' + after, encoding='utf-8') + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..d504e1e1 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/restart.py diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..063197a8 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +python scripts\stop-one.py diff --git a/scripts/stop-one.py b/scripts/stop-one.py new file mode 100644 index 00000000..839d5438 --- /dev/null +++ b/scripts/stop-one.py @@ -0,0 +1,5 @@ +import os +from ops import stop_instance + +if __name__ == '__main__': + stop_instance(os.environ['TARGET_INSTANCE_ID']) diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..911fbd6a --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/stop-one.py diff --git a/scripts/upload_load.py b/scripts/upload_load.py new file mode 100644 index 00000000..98ae701b --- /dev/null +++ b/scripts/upload_load.py @@ -0,0 +1,71 @@ +"""Upload one fixture concurrently and record the peak RSS of API containers.""" +import argparse +import json +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +def api_memory_bytes(): + text = subprocess.check_output( + ['docker', 'compose', 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', 'api'], + cwd=ROOT, text=True, + ) + values = [] + for line in text.splitlines(): + amount = line.split()[1] + number, unit = float(amount[:-3]), amount[-3:].lower() + values.append(int(number * {'b': 1, 'kib': 1024, 'mib': 1024**2, 'gib': 1024**3}[unit])) + return sum(values) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--concurrency', type=int, default=10) + parser.add_argument('--output', default='evidence/upload-load.json') + args = parser.parse_args() + check(args.concurrency >= 10, 'concurrency must be at least 10') + client = Client(args.base_url) + alice, _, _, chat = client.bootstrap() + content = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes() + start = threading.Event() + done = threading.Event() + peak = [0] + + def monitor(): + while not done.is_set(): + peak[0] = max(peak[0], api_memory_bytes()) + time.sleep(.2) + + def upload(index): + start.wait() + return Client(args.base_url, timeout=60).call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=load-%d.png' % index, + 201, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment') + + watcher = threading.Thread(target=monitor, daemon=True) + watcher.start() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + jobs = [pool.submit(upload, item) for item in range(args.concurrency)] + start.set() + results = [job.result() for job in jobs] + done.set() + watcher.join(timeout=1) + report = {'concurrency': args.concurrency, 'fixture_bytes': len(content), + 'successful_uploads': len(results), 'peak_api_memory_bytes': peak[0]} + destination = ROOT / args.output + destination.parent.mkdir(exist_ok=True) + destination.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps(report, ensure_ascii=False)) + + +if __name__ == '__main__': + main() diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 00000000..d5a34072 --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,13 @@ +import os + +import uvicorn + + +if __name__ == '__main__': + uvicorn.run( + 'src.main:app', + host=os.getenv('APP_HOST', '0.0.0.0'), + port=int(os.getenv('APP_PORT', '8080')), + workers=1, + timeout_graceful_shutdown=5, + ) diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..9f862439 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,82 @@ +import os +import unicodedata +from uuid import uuid4 + +from PIL import Image, UnidentifiedImageError +from sqlalchemy import func, select + +from .errors import APIError +from .models import Attachment, ChatMember +from .object_storage import BUCKET, PREFIX, s3_client +from .store import ensure_uuid, serialize, store + +MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760) +MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000) +if MAX_BYTES < 1 or MAX_PIXELS < 1: + raise ValueError('Upload limits must be positive') + + +def validate_filename(filename): + if (not filename or not filename.strip() or len(filename) > 255 + or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in filename)): + raise APIError(400, 'invalid_request', 'Неверное имя файла') + + +def validate_image(source, content_type): + try: + source.seek(0) + with Image.open(source, formats=['PNG', 'JPEG']) as picture: + if picture.width * picture.height > MAX_PIXELS: + raise APIError(413, 'payload_too_large', 'Слишком много пикселей') + if Image.MIME[picture.format] != content_type: + raise APIError(415, 'unsupported_media_type', 'Тип изображения не совпадает с Content-Type') + picture.verify() + source.seek(0) + with Image.open(source, formats=['PNG', 'JPEG']) as picture: + picture.load() # verify alone does not decode JPEG or detect truncated pixel data. + except Image.DecompressionBombError as exc: + raise APIError(413, 'payload_too_large', 'Слишком много пикселей') from exc + except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc: + raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from exc + source.seek(0) + + +def lock_object(session, key): + # Shared by upload and cleanup: no deletion while a COMMIT is in flight. + session.execute(select(func.pg_advisory_xact_lock(func.hashtextextended(key, 0)))) + + +def metadata(attachment): + result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename', + 'content_type', 'size_bytes', 'sha256', 'created_at') + return {**result, 'updated_at': result['created_at'], 'status': 'ready', + 'variants': [], 'error_code': None} + + +def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest): + validate_image(source, content_type) + attachment_id = str(uuid4()) + key = PREFIX + attachment_id + with store.sessions.begin() as session: + store._chat(session, chat_id, actor_id) + lock_object(session, key) + s3_client().put_object(Bucket=BUCKET, Key=key, Body=source, + ContentLength=size, ContentType=content_type) + attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id, + object_key=key, filename=filename, content_type=content_type, + size_bytes=size, sha256=digest) + session.add(attachment) + session.flush() + result = metadata(attachment) + # An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later. + return result + + +def get_attachment(attachment_id, actor_id): + with store.sessions() as session: + attachment = session.scalar(select(Attachment).join( + ChatMember, ChatMember.chat_id == Attachment.chat_id).where( + Attachment.id == ensure_uuid(attachment_id), ChatMember.user_id == actor_id)) + if attachment is None: + raise APIError(404, 'not_found', 'Объект не найден') + return attachment diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 00000000..ab93a9fd --- /dev/null +++ b/src/auth.py @@ -0,0 +1,68 @@ +import hashlib +import json +import os +import re +import secrets +from datetime import datetime, timedelta, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError +from redis import Redis + +from .errors import APIError + +password_hasher = PasswordHasher() +dummy_hash = password_hasher.hash(secrets.token_urlsafe(32)) + + +def verify_password(encoded, password): + try: + return password_hasher.verify(encoded or dummy_hash, password) and encoded is not None + except VerificationError: + return False + + +def unauthorized(): + return APIError(401, 'unauthorized', 'Нет действительной идентификации') + + +class Sessions: + def __init__(self): + self.client = Redis.from_url( + os.getenv('REDIS_URL', 'redis://redis:6379/0'), decode_responses=True, + socket_connect_timeout=1, socket_timeout=1, max_connections=20, + retry_on_timeout=False, + ) + self.ttl = int(os.getenv('SESSION_TTL_SECONDS', '3600')) + if self.ttl < 1: + raise ValueError('SESSION_TTL_SECONDS must be positive') + + @staticmethod + def key(token): + if not re.fullmatch(r'[A-Za-z0-9_-]{43}', token): + raise unauthorized() + return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest() + + def create(self, user): + token = secrets.token_urlsafe(32) + expires = datetime.now(timezone.utc) + timedelta(seconds=self.ttl) + expires_at = expires.isoformat(timespec='microseconds').replace('+00:00', 'Z') + self.client.set(self.key(token), json.dumps({'user_id': user['id'], 'expires_at': expires_at}), + pxat=int(expires.timestamp() * 1000)) + return {'token': token, 'token_type': 'Bearer', 'expires_at': expires_at, 'user': user} + + def user_id(self, token): + raw = self.client.get(self.key(token)) + if raw is None: + raise unauthorized() + session = json.loads(raw) + if datetime.fromisoformat(session['expires_at'].replace('Z', '+00:00')) <= datetime.now(timezone.utc): + raise unauthorized() + return session['user_id'] + + def delete(self, token): + if not self.client.delete(self.key(token)): + raise unauthorized() + + +sessions = Sessions() diff --git a/src/cache.py b/src/cache.py new file mode 100644 index 00000000..bde20774 --- /dev/null +++ b/src/cache.py @@ -0,0 +1,40 @@ +"""History pages are disposable; PostgreSQL owns both data and their version.""" +import hashlib +import json +import os + +from redis.exceptions import RedisError + +from .auth import sessions + + +class HistoryCache: + def __init__(self, client, ttl=30, prefix='history'): + self.client = client + self.ttl = ttl + self.prefix = prefix + if ttl < 1: + raise ValueError('CACHE_TTL_SECONDS must be positive') + + def key(self, chat_id, version, limit, after): + page = json.dumps([limit, after], separators=(',', ':')) + digest = hashlib.sha256(page.encode()).hexdigest() + return f'{self.prefix}:{chat_id}:{version}:{digest}' + + def get_or_load(self, chat_id, version, limit, after, load): + key = self.key(chat_id, version, limit, after) + try: + cached = self.client.get(key) + except RedisError: + return load() + if cached is not None: + return json.loads(cached) + page = load() + try: + self.client.set(key, json.dumps(page, ensure_ascii=False), ex=self.ttl) + except RedisError: + pass + return page + + +history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')), prefix='history-v4') diff --git a/src/cleanup.py b/src/cleanup.py new file mode 100644 index 00000000..ac1297e5 --- /dev/null +++ b/src/cleanup.py @@ -0,0 +1,61 @@ +"""Remove old S3 objects without committed metadata. Never delete on a DB error.""" +import argparse +import logging +import time +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select +from sqlalchemy.orm import sessionmaker + +from .attachments import lock_object +from .database import create_db_engine +from .models import Attachment +from .object_storage import BUCKET, PREFIX, s3_client + +GRACE = timedelta(hours=1) + + +def cleanup(sessions, client, now=None): + cutoff = (now or datetime.now(timezone.utc)) - GRACE + deleted = 0 + for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX): + for obj in page.get('Contents', []): + if obj['LastModified'] >= cutoff: + continue + key = obj['Key'] + with sessions.begin() as session: + lock_object(session, key) + if session.scalar(select(Attachment.id).where(Attachment.object_key == key)): + continue + # Recheck age after taking the same lock as the uploader. + if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff: + continue + client.delete_object(Bucket=BUCKET, Key=key) + deleted += 1 + return deleted + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--once', action='store_true') + args = parser.parse_args() + logging.basicConfig(level=logging.INFO) + engine = create_db_engine() + try: + while True: + try: + count = cleanup(sessionmaker(engine), s3_client()) + logging.info('Cleanup completed: deleted=%d', count) + except Exception as exc: + logging.error('Cleanup deferred: %s', type(exc).__name__) + if args.once: + raise SystemExit(1) from None + if args.once: + break + time.sleep(3600) + finally: + engine.dispose() + + +if __name__ == '__main__': + main() diff --git a/src/database.py b/src/database.py new file mode 100644 index 00000000..e98b4d4c --- /dev/null +++ b/src/database.py @@ -0,0 +1,51 @@ +import os +from pathlib import Path + +from sqlalchemy import create_engine, func, insert, select +from sqlalchemy.engine import URL, make_url + +from .models import SchemaMigration + + +def database_url(): + if os.getenv('DATABASE_URL'): + url = make_url(os.environ['DATABASE_URL']) + if url.drivername in ('postgres', 'postgresql'): + url = url.set(drivername='postgresql+psycopg') + if url.drivername != 'postgresql+psycopg': + raise ValueError('DATABASE_URL must use PostgreSQL with psycopg') + return url + return URL.create( + 'postgresql+psycopg', host=os.getenv('POSTGRES_HOST', 'postgres'), + database=os.getenv('POSTGRES_DB', 'messenger'), + username=os.getenv('POSTGRES_USER', 'messenger'), + password=os.environ['POSTGRES_PASSWORD'], + ) + + +def create_db_engine(): + return create_engine( + database_url(), pool_size=10, max_overflow=0, pool_timeout=2, pool_pre_ping=True, + connect_args={'connect_timeout': 2, + 'options': '-c statement_timeout=3000 -c lock_timeout=2000'}, + ) + + +def migrate(engine): + with engine.begin() as conn: + conn.execute(select(func.pg_advisory_xact_lock(202602))) + SchemaMigration.__table__.create(conn, checkfirst=True) + applied = set(conn.scalars(select(SchemaMigration.version))) + for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')): + if path.name not in applied: + conn.exec_driver_sql(path.read_text(encoding='utf-8')) + conn.execute(insert(SchemaMigration).values(version=path.name)) + + +if __name__ == '__main__': + engine = create_db_engine() + try: + migrate(engine) + print('Migrations applied') + finally: + engine.dispose() diff --git a/src/dependencies.py b/src/dependencies.py new file mode 100644 index 00000000..03aac92c --- /dev/null +++ b/src/dependencies.py @@ -0,0 +1,31 @@ +from typing import Optional + +from fastapi import Depends, Header, Query + +from .auth import sessions, unauthorized +from .errors import APIError +from .store import store + + +def page_limit(limit: str = Query(default="50")) -> int: + if not limit.isascii() or not limit.isdecimal() or len(limit) > 3: + raise APIError(400, "invalid_request", "Неверный limit") + number = int(limit) + if not 1 <= number <= 100: + raise APIError(400, "invalid_request", "Неверный limit") + return number + + + +def bearer_token(authorization: Optional[str] = Header(default=None)) -> str: + parts = (authorization or '').split() + if len(parts) != 2 or parts[0].lower() != 'bearer': + raise unauthorized() + return parts[1] + + +def current_user(token: str = Depends(bearer_token)) -> dict: + user = store.get_user(sessions.user_id(token)) + if user is None: + raise unauthorized() + return user diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..d02425bf --- /dev/null +++ b/src/errors.py @@ -0,0 +1,60 @@ +from fastapi import Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException +from .middleware import INSTANCE_ID + + +class APIError(Exception): + def __init__(self, status_code: int, code: str, message: str): + self.status_code = status_code + self.code = code + self.message = message + + +def error_response(request: Request, status_code: int, code: str, message: str) -> JSONResponse: + request_id = getattr(request.state, "request_id", "") + return JSONResponse( + status_code=status_code, + headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID}, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + ) + + +def api_error_handler(request: Request, exc: APIError) -> JSONResponse: + response = error_response(request, exc.status_code, exc.code, exc.message) + if exc.status_code == 429: + response.headers['Retry-After'] = '1' + if exc.status_code == 401: + response.headers['WWW-Authenticate'] = ( + 'Basic realm="messenger", charset="UTF-8"' + if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"' + ) + return response + + +def dependency_error_handler(request: Request, exc: Exception) -> JSONResponse: + return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно') + + +def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse: + return error_response(request, 400, "invalid_request", "Неверный запрос") + + +def http_error_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse: + if exc.status_code == 404: + return error_response(request, 404, "not_found", "Объект не найден") + if exc.status_code == 405: + return error_response(request, 400, "invalid_request", "Метод не поддерживается") + else: + return error_response(request, exc.status_code, "invalid_request", "Неверный запрос") + + +def internal_error_handler(request: Request, exc: Exception) -> JSONResponse: + return error_response(request, 500, "internal_error", "Внутренняя ошибка") diff --git a/src/limits.py b/src/limits.py new file mode 100644 index 00000000..facdf115 --- /dev/null +++ b/src/limits.py @@ -0,0 +1,99 @@ +import asyncio +import os +from dataclasses import dataclass + +from starlette.requests import Request + +from .errors import error_response +from .middleware import is_upload + + +@dataclass(frozen=True) +class Limits: + max_users: int = 1000 + max_chats: int = 1000 + max_messages: int = 10000 + max_text_bytes: int = 16 * 1024 * 1024 + max_request_bytes: int = 64 * 1024 + max_inflight: int = 32 + message_rate: int = 1000 + message_burst: int = 1000 + + @classmethod + def from_env(cls): + values = {} + for name, default in cls().__dict__.items(): + value = int(os.getenv(name.upper(), str(default))) + if value < 1: + raise ValueError(f'{name.upper()} must be positive') + values[name] = value + return cls(**values) + + +limits = Limits.from_env() + + +class ResourceLimitMiddleware: + def __init__(self, app, settings=limits): + self.app = app + self.settings = settings + self.active = 0 + + async def __call__(self, scope, receive, send): + if scope['type'] != 'http' or scope['path'].startswith('/health/'): + return await self.app(scope, receive, send) + + request = Request(scope) + + async def reject(status, code, message): + response = error_response(request, status, code, message) + if status == 503: + response.headers['Retry-After'] = '1' + await response(scope, receive, send) + + if self.active >= self.settings.max_inflight: + return await reject(503, 'unavailable', 'Слишком много одновременных запросов') + self.active += 1 + try: + # The upload endpoint authenticates and bounds its stream itself. + if is_upload(scope['method'], scope['path']): + return await self.app(scope, receive, send) + length = request.headers.get('content-length') + if length is not None: + try: + size = int(length) + if size < 0: + raise ValueError() + except ValueError: + return await reject(400, 'invalid_request', 'Неверный Content-Length') + if size > self.settings.max_request_bytes: + return await reject(413, 'payload_too_large', 'Тело запроса слишком большое') + + body = bytearray() + try: + async with asyncio.timeout(5): + while True: + message = await receive() + if message['type'] == 'http.disconnect': + return + chunk = message.get('body', b'') + if len(body) + len(chunk) > self.settings.max_request_bytes: + return await reject(413, 'payload_too_large', 'Тело запроса слишком большое') + body.extend(chunk) + if not message.get('more_body', False): + break + except TimeoutError: + return await reject(503, 'unavailable', 'Истекло время чтения запроса') + + delivered = False + + async def buffered_receive(): + nonlocal delivered + if not delivered: + delivered = True + return {'type': 'http.request', 'body': bytes(body), 'more_body': False} + return await receive() + + await self.app(scope, buffered_receive, send) + finally: + self.active -= 1 diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..fc96b189 --- /dev/null +++ b/src/main.py @@ -0,0 +1,48 @@ +from fastapi import FastAPI +from fastapi.exceptions import RequestValidationError +from starlette.exceptions import HTTPException +from contextlib import asynccontextmanager +from sqlalchemy.exc import OperationalError, InterfaceError, TimeoutError as PoolTimeout +from sqlalchemy.orm import sessionmaker +from redis.exceptions import RedisError +from botocore.exceptions import BotoCoreError, ClientError + +from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler, dependency_error_handler +from .database import create_db_engine +from .store import store +from .auth import sessions +from .middleware import RequestIdMiddleware +from .limits import ResourceLimitMiddleware +from .routers.health import router as health_router +from .routers.users import router as users_router +from .routers.chats import router as chats_router +from .routers.auth import router as auth_router +from .routers.attachments import router as attachments_router + + +@asynccontextmanager +async def lifespan(app): + engine = create_db_engine() + try: + store.sessions = sessionmaker(engine, expire_on_commit=False) + yield + finally: + engine.dispose() + sessions.client.close() + + +app = FastAPI(title="MTUSI Messenger", lifespan=lifespan) +app.add_middleware(ResourceLimitMiddleware) +app.add_middleware(RequestIdMiddleware) +app.add_exception_handler(APIError, api_error_handler) +app.add_exception_handler(RequestValidationError, validation_error_handler) +app.add_exception_handler(HTTPException, http_error_handler) +app.add_exception_handler(Exception, internal_error_handler) +for error in (OperationalError, InterfaceError, PoolTimeout, RedisError, BotoCoreError, ClientError): + app.add_exception_handler(error, dependency_error_handler) + +app.include_router(health_router) +app.include_router(users_router) +app.include_router(chats_router) +app.include_router(auth_router) +app.include_router(attachments_router) diff --git a/src/middleware.py b/src/middleware.py new file mode 100644 index 00000000..c2b73a16 --- /dev/null +++ b/src/middleware.py @@ -0,0 +1,43 @@ +from uuid import uuid4 +from socket import gethostname +from starlette.responses import JSONResponse +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +import re + + +def is_upload(method, path): + return method == 'POST' and re.fullmatch(r'/api/v1/chats/[^/]+/attachments', path) is not None + +INSTANCE_ID = gethostname() + + +class RequestIdMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + request_id = request.headers.get("X-Request-Id", "").strip() or str(uuid4()) + request.state.request_id = request_id + + if (request.method in {"POST", "PUT", "PATCH"} + and request.url.path.startswith("/api/") + and not is_upload(request.method, request.url.path) + and request.url.path != '/api/v1/auth/sessions'): + content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if content_type != "application/json": + response = JSONResponse( + status_code=415, + content={ + "error": { + "code": "unsupported_media_type", + "message": "Ожидается Content-Type application/json", + "request_id": request_id, + } + }, + ) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + response = await call_next(request) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response diff --git a/src/models.py b/src/models.py new file mode 100644 index 00000000..7e1582b9 --- /dev/null +++ b/src/models.py @@ -0,0 +1,85 @@ +from datetime import datetime +from uuid import uuid4 + +from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +from sqlalchemy.dialects.postgresql import ARRAY + + +class Base(DeclarativeBase): + pass + + +class User(Base): + __tablename__ = 'users' + __table_args__ = ( + CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'"), + CheckConstraint('char_length(display_name) BETWEEN 1 AND 100'), + ) + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + username: Mapped[str] = mapped_column(Text, unique=True) + display_name: Mapped[str] = mapped_column(Text) + password_hash: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + + +class Chat(Base): + __tablename__ = 'chats' + __table_args__ = ( + CheckConstraint('char_length(title) BETWEEN 1 AND 100'), + Index('chats_order_idx', 'created_at', 'id'), + ) + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + title: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + history_version: Mapped[int] = mapped_column(BigInteger, server_default='0') + + +class ChatMember(Base): + __tablename__ = 'chat_members' + __table_args__ = (Index('chat_members_user_idx', 'user_id', 'chat_id'),) + chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True) + user_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True) + + +class Message(Base): + __tablename__ = 'messages' + __table_args__ = ( + CheckConstraint('char_length(text) <= 4000'), + CheckConstraint("length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0"), + Index('messages_history_idx', 'chat_id', 'created_at', 'id'), + ) + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE')) + sender_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id')) + text: Mapped[str] = mapped_column(Text) + attachment_ids: Mapped[list[str]] = mapped_column(ARRAY(Uuid(as_uuid=False)), default=list, server_default='{}') + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + + +class SchemaMigration(Base): + __tablename__ = 'schema_migrations' + version: Mapped[str] = mapped_column(Text, primary_key=True) + applied_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=True) + + +class Attachment(Base): + __tablename__ = 'attachments' + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True) + chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE')) + owner_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id')) + object_key: Mapped[str] = mapped_column(Text, unique=True) + filename: Mapped[str] = mapped_column(Text) + content_type: Mapped[str] = mapped_column(Text) + size_bytes: Mapped[int] = mapped_column(BigInteger) + sha256: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + + +class IdempotencyRecord(Base): + __tablename__ = 'message_idempotency' + actor_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True) + chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True) + key: Mapped[str] = mapped_column(Text, primary_key=True) + message_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('messages.id', ondelete='CASCADE')) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) diff --git a/src/object_storage.py b/src/object_storage.py new file mode 100644 index 00000000..d3cb2527 --- /dev/null +++ b/src/object_storage.py @@ -0,0 +1,23 @@ +import os +from functools import lru_cache + +import boto3 +from botocore.config import Config + +BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments') +PREFIX = 'originals/' + + +@lru_cache(maxsize=1) +def s3_client(): + return boto3.client( + 's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'), + region_name=os.getenv('S3_REGION', 'us-east-1'), + aws_access_key_id=os.environ['S3_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['S3_SECRET_ACCESS_KEY'], + config=Config(connect_timeout=2, read_timeout=5, + retries={'total_max_attempts': 1}, + s3={'addressing_style': 'path'}, + request_checksum_calculation='when_required', + response_checksum_validation='when_required'), + ) diff --git a/src/routers/__init__.py b/src/routers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/routers/attachments.py b/src/routers/attachments.py new file mode 100644 index 00000000..57007b6f --- /dev/null +++ b/src/routers/attachments.py @@ -0,0 +1,84 @@ +import asyncio +import hashlib +from tempfile import TemporaryFile +from urllib.parse import quote + +from fastapi import APIRouter, Depends, Query, Request +from starlette.background import BackgroundTask +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect +from starlette.responses import StreamingResponse + +from ..attachments import MAX_BYTES, get_attachment, metadata, save_attachment, validate_filename +from ..dependencies import current_user +from ..errors import APIError +from ..object_storage import BUCKET, s3_client +from ..store import ensure_uuid, store + +router = APIRouter(prefix='/api/v1', tags=['attachments']) +# At most one decoded image (<=40M pixels) per API process; incoming bodies use disk. +decode_slot = asyncio.Semaphore(1) + + +@router.post('/chats/{chat_id}/attachments', status_code=201) +async def upload(chat_id: str, request: Request, filename: str = Query(...), + user: dict = Depends(current_user)): + chat_id = ensure_uuid(chat_id) + await run_in_threadpool(store.get_chat_for_user, chat_id, user['id']) + validate_filename(filename) + content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower() + if content_type not in ('image/png', 'image/jpeg'): + raise APIError(415, 'unsupported_media_type', 'Ожидается PNG или JPEG') + length = request.headers.get('content-length') + if length is not None: + try: + length = int(length) + if length < 0: + raise ValueError() + except ValueError as exc: + raise APIError(400, 'invalid_request', 'Неверный Content-Length') from exc + if length > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Файл слишком большой') + size = 0 + digest = hashlib.sha256() + with TemporaryFile() as source: + try: + async with asyncio.timeout(30): + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Файл слишком большой') + digest.update(chunk) + await run_in_threadpool(source.write, chunk) + except (TimeoutError, ClientDisconnect) as exc: + raise APIError(400, 'invalid_request', 'Загрузка прервана') from exc + if length is not None and size != length: + raise APIError(400, 'invalid_request', 'Неполное тело запроса') + async with decode_slot: + return await run_in_threadpool(save_attachment, source, chat_id, user['id'], + filename, content_type, size, digest.hexdigest()) + + +@router.get('/attachments/{attachment_id}') +def attachment_info(attachment_id: str, user: dict = Depends(current_user)): + return metadata(get_attachment(attachment_id, user['id'])) + + +@router.get('/attachments/{attachment_id}/content') +def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)): + attachment = get_attachment(attachment_id, user['id']) + if variant != 'original': + raise APIError(404, 'not_found', 'Вариант не найден') + body = s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body'] + + def chunks(): + try: + yield from body.iter_chunks(chunk_size=64 * 1024) + finally: + body.close() + + return StreamingResponse(chunks(), media_type=attachment.content_type, + headers={'Content-Length': str(attachment.size_bytes), + 'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''), + 'X-Content-Type-Options': 'nosniff'}, + background=BackgroundTask(body.close)) diff --git a/src/routers/auth.py b/src/routers/auth.py new file mode 100644 index 00000000..bb4beaa4 --- /dev/null +++ b/src/routers/auth.py @@ -0,0 +1,34 @@ +import base64 + +from fastapi import APIRouter, Depends, Header, Request, Response +from starlette.concurrency import run_in_threadpool + +from ..auth import sessions, unauthorized +from ..dependencies import bearer_token, current_user +from ..errors import APIError +from ..store import store + +router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth']) + + +@router.post('', status_code=201) +async def create_session(request: Request, authorization: str = Header(default='')): + if await request.body(): + raise APIError(400, 'invalid_request', 'Вход не принимает тело запроса') + try: + scheme, encoded = authorization.split() + if scheme.lower() != 'basic': + raise ValueError() + username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + if not username.isascii() or not 12 <= len(password) <= 128 or '\x00' in username: + raise ValueError() + except (ValueError, UnicodeError): + raise unauthorized() from None + user = await run_in_threadpool(store.login, username, password) + return await run_in_threadpool(sessions.create, user) + + +@router.delete('/current', status_code=204) +def logout(token: str = Depends(bearer_token), user: dict = Depends(current_user)): + sessions.delete(token) + return Response(status_code=204) diff --git a/src/routers/chats.py b/src/routers/chats.py new file mode 100644 index 00000000..58aa1c72 --- /dev/null +++ b/src/routers/chats.py @@ -0,0 +1,49 @@ +from typing import Optional +import re + +from fastapi import APIRouter, Depends, Header, Query, status +from ..errors import APIError + +from ..dependencies import current_user, page_limit +from ..schemas.chat import Chat, ChatPage, CreateChat +from ..schemas.message import CreateMessage, Message, MessagePage +from ..store import store + +router = APIRouter(prefix="/api/v1/chats", tags=["chats"]) + + +@router.post("", response_model=Chat, status_code=status.HTTP_201_CREATED) +def create_chat(payload: CreateChat, user: dict = Depends(current_user)): + return store.create_chat(user["id"], payload.title, payload.member_ids) + + +@router.get("", response_model=ChatPage) +def list_chats( + limit: int = Depends(page_limit), + cursor: Optional[str] = Query(default=None, min_length=1), + user: dict = Depends(current_user), +): + return store.list_chats(user["id"], limit, cursor) + + +@router.get("/{chat_id}", response_model=Chat) +def get_chat(chat_id: str, user: dict = Depends(current_user)): + return store.get_chat_for_user(chat_id, user["id"]) + + +@router.post("/{chat_id}/messages", response_model=Message, status_code=status.HTTP_201_CREATED) +def create_message(chat_id: str, payload: CreateMessage, user: dict = Depends(current_user), + idempotency_key: Optional[str] = Header(default=None)): + if not idempotency_key or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', idempotency_key): + raise APIError(400, 'invalid_request', 'Требуется корректный Idempotency-Key') + return store.create_message(chat_id, user["id"], payload.text, idempotency_key, payload.attachment_ids) + + +@router.get("/{chat_id}/messages", response_model=MessagePage) +def list_messages( + chat_id: str, + limit: int = Depends(page_limit), + cursor: Optional[str] = Query(default=None, min_length=1), + user: dict = Depends(current_user), +): + return store.list_messages(chat_id, user["id"], limit, cursor) diff --git a/src/routers/health.py b/src/routers/health.py new file mode 100644 index 00000000..f2015d38 --- /dev/null +++ b/src/routers/health.py @@ -0,0 +1,23 @@ +from fastapi import APIRouter +from sqlalchemy import select + +from ..schemas.common import Health +from ..auth import sessions +from ..store import store +from ..object_storage import BUCKET, s3_client + +router = APIRouter(prefix='/health') + + +@router.get("/live", response_model=Health) +def health_live(): + return {"status": "ok"} + + +@router.get("/ready", response_model=Health) +def health_ready(): + with store.sessions() as session: + session.execute(select(1)) + sessions.client.ping() + s3_client().head_bucket(Bucket=BUCKET) + return {"status": "ok"} diff --git a/src/routers/users.py b/src/routers/users.py new file mode 100644 index 00000000..2c8b69c9 --- /dev/null +++ b/src/routers/users.py @@ -0,0 +1,19 @@ +from fastapi import APIRouter, Depends, status + +from ..dependencies import current_user +from ..schemas.user import CreateUser, User +from ..store import store + + + +router = APIRouter(prefix="/api/v1/users", tags=["users"]) + + +@router.post("", response_model=User, status_code=status.HTTP_201_CREATED) +def create_user(payload: CreateUser): + return store.create_user(payload.username, payload.display_name, payload.password) + + +@router.get("/me", response_model=User) +def get_current_user(user: dict = Depends(current_user)): + return user diff --git a/src/schemas/__init__.py b/src/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/schemas/chat.py b/src/schemas/chat.py new file mode 100644 index 00000000..b1a3629d --- /dev/null +++ b/src/schemas/chat.py @@ -0,0 +1,35 @@ +from typing import List, Optional + +from pydantic import Field, StrictStr, field_validator + +from .common import ContractModel + + + +class CreateChat(ContractModel): + title: StrictStr = Field(min_length=1, max_length=100) + member_ids: List[StrictStr] = Field(min_length=1, max_length=100) + + @field_validator("title") + def validate_title(cls, value): + if not value.strip(): + raise ValueError("blank title") + return value + + @field_validator("member_ids") + def validate_member_ids(cls, value): + if len(value) != len(set(value)): + raise ValueError("duplicate member_ids") + return value + + +class Chat(ContractModel): + id: str + title: str + member_ids: List[str] + created_at: str + + +class ChatPage(ContractModel): + items: List[Chat] + next_cursor: Optional[str] diff --git a/src/schemas/common.py b/src/schemas/common.py new file mode 100644 index 00000000..06a7bc20 --- /dev/null +++ b/src/schemas/common.py @@ -0,0 +1,17 @@ +from pydantic import BaseModel, ConfigDict, field_validator + + +class ContractModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + @field_validator('*') + def validate_text_encoding(cls, value, info): + if isinstance(value, str): + value.encode('utf-8') + if info.field_name != 'password' and '\x00' in value: + raise ValueError('NUL is not supported in PostgreSQL text') + return value + + +class Health(ContractModel): + status: str diff --git a/src/schemas/message.py b/src/schemas/message.py new file mode 100644 index 00000000..75268c55 --- /dev/null +++ b/src/schemas/message.py @@ -0,0 +1,39 @@ +from typing import List, Optional + +from pydantic import Field, StrictStr, field_validator, model_validator +from uuid import UUID + +from .common import ContractModel + + + +class CreateMessage(ContractModel): + text: StrictStr = Field(max_length=4000) + attachment_ids: List[StrictStr] = Field(default_factory=list, max_length=10) + + @field_validator('attachment_ids') + def validate_ids(cls, value): + ids = [str(UUID(item)) for item in value] + if len(ids) != len(set(ids)): + raise ValueError('duplicate attachments') + return ids + + @model_validator(mode='after') + def validate_content(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError('empty message') + return self + + +class Message(ContractModel): + id: str + chat_id: str + sender_id: str + text: str + attachment_ids: List[str] + created_at: str + + +class MessagePage(ContractModel): + items: List[Message] + next_cursor: Optional[str] diff --git a/src/schemas/user.py b/src/schemas/user.py new file mode 100644 index 00000000..bd18d5bc --- /dev/null +++ b/src/schemas/user.py @@ -0,0 +1,34 @@ +import re + +from pydantic import Field, StrictStr, field_validator + +from .common import ContractModel + + + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + + +class CreateUser(ContractModel): + username: StrictStr + display_name: StrictStr = Field(min_length=1, max_length=100) + password: StrictStr = Field(min_length=12, max_length=128) + + @field_validator("username") + def validate_username(cls, value): + if not USERNAME_RE.fullmatch(value): + raise ValueError("invalid username") + return value + + @field_validator("display_name") + def validate_display_name(cls, value): + if not value.strip(): + raise ValueError("blank display_name") + return value + + +class User(ContractModel): + id: str + username: str + display_name: str = Field(min_length=1, max_length=100) + created_at: str diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..48d2a225 --- /dev/null +++ b/src/store.py @@ -0,0 +1,226 @@ +import base64 +import json +from datetime import datetime, timezone +from typing import Tuple +from uuid import UUID + +from sqlalchemy import select, tuple_, update +from sqlalchemy.exc import IntegrityError + +from .auth import password_hasher, verify_password, unauthorized +from .errors import APIError +from .cache import history_cache +from .models import User, Chat, ChatMember, Message, IdempotencyRecord, Attachment + + +def ensure_uuid(value): + try: + if not isinstance(value, str): + raise ValueError() + return str(UUID(value)) + except (TypeError, ValueError) as exc: + raise APIError(400, 'invalid_request', 'Неверный UUID') from exc + + +def serialize(model, *fields): + if model is None: + return None + result = {} + for field in fields: + value = getattr(model, field) + if isinstance(value, datetime): + value = value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + result[field] = value + return result + + +USER_FIELDS = ('id', 'username', 'display_name', 'created_at') +CHAT_FIELDS = ('id', 'title', 'created_at') +MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at', 'attachment_ids') + + +def after_condition(model, after): + return tuple_(model.created_at, model.id) > ( + datetime.fromisoformat(after[0].replace('Z', '+00:00')), after[1]) + + +def history_query(chat_id, limit, after): + query = select(Message).where(Message.chat_id == chat_id) + if after: + query = query.where(after_condition(Message, after)) + return query.order_by(Message.created_at, Message.id).limit(limit + 1) + + +class PostgresStore: + def __init__(self, cache=history_cache): + self.sessions = None + self.cache = cache + + def create_user(self, username, display_name, password): + user = User(username=username, display_name=display_name, password_hash=password_hasher.hash(password)) + try: + with self.sessions.begin() as session: + session.add(user) + session.flush() + result = serialize(user, *USER_FIELDS) + return result + except IntegrityError as exc: + if getattr(exc.orig, 'sqlstate', None) == '23505': + raise APIError(409, 'conflict', 'Имя пользователя уже занято') from exc + raise + + def get_user(self, user_id): + with self.sessions() as session: + return serialize(session.get(User, ensure_uuid(user_id)), *USER_FIELDS) + + def login(self, username, password): + with self.sessions() as session: + user = session.scalar(select(User).where(User.username == username)) + encoded = user.password_hash if user else None + result = serialize(user, *USER_FIELDS) + if not verify_password(encoded, password): + raise unauthorized() + return result + + def create_chat(self, actor_id, title, member_ids): + members = [ensure_uuid(member) for member in member_ids] + if len(members) != len(set(members)) or actor_id not in members: + raise APIError(400, 'invalid_request', 'Неверный список участников') + try: + with self.sessions.begin() as session: + chat = Chat(title=title) + session.add(chat) + session.flush() + session.add_all([ChatMember(chat_id=chat.id, user_id=member) for member in members]) + result = serialize(chat, *CHAT_FIELDS) + result['member_ids'] = sorted(members) + return result + except IntegrityError as exc: + if getattr(exc.orig, 'sqlstate', None) == '23503': + raise APIError(400, 'invalid_request', 'Неизвестный участник') from exc + raise + + @staticmethod + def _chat(session, chat_id, actor_id): + chat = session.scalar(select(Chat).join(ChatMember).where( + Chat.id == chat_id, ChatMember.user_id == actor_id)) + if chat is None: + raise APIError(404, 'not_found', 'Объект не найден') + return chat + + @staticmethod + def _members(session, chat): + result = serialize(chat, *CHAT_FIELDS) + result['member_ids'] = list(session.scalars(select(ChatMember.user_id).where( + ChatMember.chat_id == chat.id).order_by(ChatMember.user_id))) + return result + + def get_chat_for_user(self, chat_id, actor_id): + with self.sessions() as session: + return self._members(session, self._chat(session, ensure_uuid(chat_id), actor_id)) + + def list_chats(self, actor_id, limit, cursor): + after = self._decode_cursor(cursor, 'chats', actor_id) if cursor else None + query = select(Chat).join(ChatMember).where(ChatMember.user_id == actor_id) + if after: + query = query.where(after_condition(Chat, after)) + query = query.order_by(Chat.created_at, Chat.id).limit(limit + 1) + with self.sessions() as session: + chats = list(session.scalars(query)) + items = [self._members(session, chat) for chat in chats[:limit]] + return self._page(items, len(chats) > limit, 'chats', actor_id) + + def create_message(self, chat_id, actor_id, text, idempotency_key=None, attachment_ids=None): + chat_id = ensure_uuid(chat_id) + attachment_ids = [ensure_uuid(item) for item in (attachment_ids or [])] + with self.sessions.begin() as session: + self._chat(session, chat_id, actor_id) + if attachment_ids: + found = list(session.scalars(select(Attachment.id).where( + Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id))) + if len(found) != len(attachment_ids): + raise APIError(404, 'not_found', 'Объект не найден') + if idempotency_key is not None: + # Serialize writers to this chat before looking up the key. + # The same row is updated below for history cache invalidation. + session.execute(select(Chat.id).where(Chat.id == chat_id).with_for_update()) + record = session.get(IdempotencyRecord, (actor_id, chat_id, idempotency_key)) + if record is not None: + previous = session.get(Message, record.message_id) + if previous.text != text or previous.attachment_ids != attachment_ids: + raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения') + return serialize(previous, *MESSAGE_FIELDS) + session.execute(update(Chat).where(Chat.id == chat_id).values( + history_version=Chat.history_version + 1)) + message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids) + session.add(message) + session.flush() + if idempotency_key is not None: + session.add(IdempotencyRecord(actor_id=actor_id, chat_id=chat_id, + key=idempotency_key, message_id=message.id)) + result = serialize(message, *MESSAGE_FIELDS) + return result + + def list_messages(self, chat_id, actor_id, limit, cursor): + chat_id = ensure_uuid(chat_id) + after = self._decode_cursor(cursor, 'messages', chat_id) if cursor else None + with self.sessions() as session: + version = self._chat(session, chat_id, actor_id).history_version + return self.cache.get_or_load( + chat_id, version, limit, after, + lambda: self._load_messages(chat_id, limit, after), + ) + + def _load_messages(self, chat_id, limit, after): + with self.sessions() as session: + messages = list(session.scalars(history_query(chat_id, limit, after))) + items = [serialize(message, *MESSAGE_FIELDS) for message in messages[:limit]] + return self._page(items, len(messages) > limit, 'messages', chat_id) + + def _page(self, items, has_more, resource, owner_id): + cursor = None + if has_more: + last = items[-1] + cursor = self._encode_cursor(resource, owner_id, last['created_at'], last['id']) + return {'items': items, 'next_cursor': cursor} + + @staticmethod + def _encode_cursor(resource: str, owner_id: str, created_at: str, item_id: str) -> str: + payload = { + "resource": resource, + "owner_id": owner_id, + "created_at": created_at, + "id": item_id, + } + raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + @staticmethod + def _decode_cursor(cursor: str, resource: str, owner_id: str) -> Tuple[str, str]: + try: + padded = cursor + "=" * (-len(cursor) % 4) + payload = json.loads(base64.b64decode(padded.encode("ascii"), altchars=b"-_", validate=True).decode("utf-8")) + except (ValueError, UnicodeError, json.JSONDecodeError) as exc: + raise APIError(400, "invalid_request", "Неверный cursor") from exc + + if not isinstance(payload, dict): + raise APIError(400, "invalid_request", "Неверный cursor") + + if payload.get("resource") != resource or payload.get("owner_id") != owner_id: + raise APIError(400, "invalid_request", "Неверный cursor") + + item_id = ensure_uuid(payload.get("id")) + created_at = payload.get("created_at") + if not isinstance(created_at, str) or not created_at: + raise APIError(400, "invalid_request", "Неверный cursor") + try: + parsed = datetime.fromisoformat(created_at.replace("Z", "+00:00")) + if not created_at.endswith("Z") or parsed.tzinfo != timezone.utc: + raise ValueError("Expected UTC timestamp") + created_at = parsed.isoformat(timespec="microseconds").replace("+00:00", "Z") + except ValueError as exc: + raise APIError(400, "invalid_request", "Неверный cursor") from exc + return created_at, item_id + + +store = PostgresStore() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..c23cf967 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,39 @@ +"""Attachment negative cases. Run against the Compose API at localhost:8080.""" +import sys +from pathlib import Path +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, check + + +def main(): + client = Client() + alice, bob, eve, chat = client.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png' + image = (Path(__file__).resolve().parents[1] / 'tests' / 'fixtures' / 'sample.png').read_bytes() + + client.call('POST', path, 415, actor=alice, raw=b'not an image', + headers={'Content-Type': 'image/png'}) + client.call('POST', path, 415, actor=alice, raw=image, + headers={'Content-Type': 'image/jpeg'}) + client.call('POST', path, 413, actor=alice, raw=b'x' * (10 * 1024 * 1024 + 1), + headers={'Content-Type': 'image/png'}) + client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=../sample.png', + 400, actor=alice, raw=image, headers={'Content-Type': 'image/png'}) + + attachment = client.call('POST', path, 201, actor=alice, raw=image, + headers={'Content-Type': 'image/png'}, schema='Attachment')['json'] + client.call('GET', '/api/v1/attachments/' + attachment['id'], 404, actor=eve) + client.call('GET', '/api/v1/attachments/' + str(uuid4()), 404, actor=alice) + + other = client.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Other chat', 'member_ids': [alice['id']]}, schema='Chat')['json'] + client.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=alice, + body={'text': '', 'attachment_ids': [attachment['id']]}, + headers={'Idempotency-Key': uuid4().hex}) + + +if __name__ == '__main__': + main() + print('PASS: attachment validation and access checks') diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..59362b4b --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,55 @@ +"""Database atomicity and concurrent conflict checks, using isolated synthetic data.""" +import unittest +from concurrent.futures import ThreadPoolExecutor +from unittest.mock import patch +from uuid import uuid4 + +from sqlalchemy import func, select +from sqlalchemy.orm import Session, sessionmaker + +from src.database import create_db_engine +from src.errors import APIError +from src.models import Chat, IdempotencyRecord, Message +from src.store import PostgresStore + + +class IdempotencyTests(unittest.TestCase): + def setUp(self): + self.engine = create_db_engine() + self.store = PostgresStore() + self.store.sessions = sessionmaker(self.engine, expire_on_commit=False) + self.user = self.store.create_user('idem_' + uuid4().hex[:16], 'Idempotency', 'Synthetic-password-123') + self.chat = self.store.create_chat(self.user['id'], 'Atomic', [self.user['id']]) + + def tearDown(self): + self.engine.dispose() + + def test_record_and_message_rollback_together(self): + real_add = Session.add + def fail_record(session, instance, *args, **kwargs): + if isinstance(instance, IdempotencyRecord): + raise RuntimeError('Simulate failure after message INSERT, before COMMIT') + return real_add(session, instance, *args, **kwargs) + args = (self.chat['id'], self.user['id'], 'atomic', 'same-key') + with patch.object(Session, 'add', fail_record): + with self.assertRaises(RuntimeError): + self.store.create_message(*args) + with Session(self.engine) as session: + self.assertEqual(session.scalar(select(func.count()).select_from(Message).where(Message.chat_id == self.chat['id'])), 0) + self.assertIsNone(session.get(IdempotencyRecord, (self.user['id'], self.chat['id'], 'same-key'))) + self.assertEqual(session.get(Chat, self.chat['id']).history_version, 0) + first = self.store.create_message(*args) + self.assertEqual(self.store.create_message(*args), first) + + def test_concurrent_different_payloads_have_one_winner(self): + def send(index): + try: + return self.store.create_message(self.chat['id'], self.user['id'], str(index), 'conflict'), 201 + except APIError as exc: + return None, exc.status_code + with ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(send, range(8))) + self.assertEqual(sorted(code for _, code in results), [201] + [409] * 7) + with Session(self.engine) as session: + self.assertEqual(session.scalar(select(func.count()).select_from(Message).where(Message.chat_id == self.chat['id'])), 1) + self.assertEqual(session.get(Chat, self.chat['id']).history_version, 1) diff --git a/student-tests/test_lab2.py b/student-tests/test_lab2.py new file mode 100644 index 00000000..bda481f8 --- /dev/null +++ b/student-tests/test_lab2.py @@ -0,0 +1,127 @@ +"""Run inside the API container: python -m unittest discover -s student-tests -v.""" +import base64 +import os +import subprocess +import sys +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from uuid import uuid4 +from sqlalchemy import func, select +from sqlalchemy.orm import sessionmaker + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client +from src.auth import sessions +from src.database import create_db_engine, migrate +from src.models import User, Chat, ChatMember, SchemaMigration + + +class Lab2Tests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.engine = create_db_engine() + cls.sessions = sessionmaker(cls.engine) + + @classmethod + def tearDownClass(cls): + cls.engine.dispose() + + def setUp(self): + self.client = Client('http://127.0.0.1:8080') + + def test_registration_race(self): + body = {'username': 'race_' + uuid4().hex[:12], 'display_name': 'Race', + 'password': 'Synthetic-password-123'} + def register(_): + return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status'] + with ThreadPoolExecutor(max_workers=4) as executor: + codes = list(executor.map(register, range(4))) + self.assertEqual(sorted(codes), [201, 409, 409, 409]) + with self.sessions() as session: + count = session.scalar(select(func.count()).select_from(User).where(User.username == body['username'])) + self.assertEqual(count, 1) + + def test_chat_rollback(self): + actor = self.client.user('rollback') + title = 'rollback_' + uuid4().hex + self.client.call('POST', '/api/v1/chats', 400, actor=actor, + body={'title': title, 'member_ids': [actor['id'], str(uuid4())]}) + with self.sessions() as session: + self.assertEqual(session.scalar(select(func.count()).select_from(Chat).where(Chat.title == title)), 0) + self.assertEqual(session.scalar(select(func.count()).select_from(ChatMember).where( + ChatMember.user_id == actor['id'])), 0) + + def test_password_logout_and_identity(self): + actor = self.client.user('auth') + credential = base64.b64encode((actor['user']['username'] + ':Wrong-password-123').encode()).decode() + self.client.call('POST', '/api/v1/auth/sessions', 401, + headers={'Authorization': 'Basic ' + credential}) + self.client.call('GET', '/api/v1/users/me', 401, headers={'Authorization': 'Basic ' + credential}) + self.client.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': actor['id']}) + me = self.client.call('GET', '/api/v1/users/me', actor=actor, + headers={'X-User-Id': str(uuid4())}, schema='User')['json'] + self.assertEqual(me['id'], actor['id']) + ttl = sessions.client.ttl(sessions.key(actor['token'])) + self.assertGreater(ttl, 0) + self.assertLessEqual(ttl, 3600) + self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor) + self.client.call('GET', '/api/v1/users/me', 401, actor=actor) + self.client.call('DELETE', '/api/v1/auth/sessions/current', 401, actor=actor) + self.assertIsNone(sessions.client.get(sessions.key(actor['token']))) + with self.sessions() as session: + encoded = session.get(User, actor['id']).password_hash + self.assertTrue(encoded.startswith('$argon2id$')) + self.assertNotIn(actor['registration']['password'], encoded) + + def test_unicode_password(self): + username = 'unicode_' + uuid4().hex[:12] + password = '🙂:я' * 40 + self.client.call('POST', '/api/v1/users', 201, + body={'username': username, 'display_name': 'Unicode', 'password': password}) + credential = base64.b64encode((username + ':' + password).encode()).decode() + result = self.client.call('POST', '/api/v1/auth/sessions', 201, + headers={'Authorization': 'Basic ' + credential}, schema='Session')['json'] + self.assertEqual(result['user']['username'], username) + + def test_migrations_repeat(self): + with self.sessions() as session: + before = session.scalar(select(func.count()).select_from(User)) + migrate(self.engine) + migrate(self.engine) + with self.sessions() as session: + self.assertEqual(session.scalar(select(func.count()).select_from(User)), before) + versions = set(session.scalars(select(SchemaMigration.version))) + expected = {path.name for path in (Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')} + self.assertEqual(versions, expected) + + def test_expiry_short_ttl(self): + process = subprocess.Popen( + [sys.executable, '-m', 'src'], + env={**os.environ, 'SESSION_TTL_SECONDS': '2', 'APP_PORT': '8081', 'APP_HOST': '127.0.0.1'}, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + try: + client = Client('http://127.0.0.1:8081') + for _ in range(100): + try: + client.call('GET', '/health/ready') + break + except AssertionError: + self.assertIsNone(process.poll(), 'Short-TTL API exited') + time.sleep(0.1) + else: + self.fail('Short-TTL API did not become ready') + actor = client.user('expiry') + client.call('GET', '/api/v1/users/me', actor=actor, schema='User') + time.sleep(2.2) + client.call('GET', '/api/v1/users/me', 401, actor=actor) + self.assertIsNone(sessions.client.get(sessions.key(actor['token']))) + finally: + process.terminate() + process.wait(timeout=10) + + +if __name__ == '__main__': + unittest.main()