diff --git a/.env.example b/.env.example index d99d26a2..b0f3dd05 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,25 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 +# Скопируйте в .env. Файл .env не коммитить. +# Пароли ниже — учебные заглушки локального стенда, не боевые секреты. APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +SESSION_STORE=valkey +POSTGRES_USER=messenger +POSTGRES_PASSWORD=local-only-change-me +POSTGRES_DB=messenger API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 +DRAIN_SECONDS=3 +# Не задавайте INSTANCE_ID: иначе все реплики получат один X-Instance-Id. +# DATABASE_URL и REDIS_URL собирает compose.yaml. +# Если задан DATABASE_URL_FILE или CACHE_URL_FILE и файл существует, приложение читает его. S3_ENDPOINT=http://s3:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +# Ключ доступа без символа «/»: его требует SigV4. Секрет — отдельная строка. +S3_ACCESS_KEY_ID=LAB4LOCALACCESS +S3_SECRET_ACCESS_KEY=local-only-change-me-s3 +# Необязательные файлы вместо переменных выше, если путь существует: +# S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id +# S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..04901310 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +*.sh text eol=lf +*.cfg text eol=lf +compose.yaml text eol=lf +Dockerfile text eol=lf +.env.example text eol=lf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..91f120fe 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Самсонов Александр, БВТ2303 +Целевая оценка (3/4/5): 3 +Трек 5, если заявлен: нет -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: стенд lab3 (Traefik, две реплики, PostgreSQL, Valkey, идемпотентность) плюс приватный bucket RustFS 1.0.4 и синхронные вложения PNG/JPEG. +Команды запуска из чистого clone: см. REPORT.md +Результаты проверок и ссылка на REPORT.md: smoke, persistence и failover PASS; отчёт REPORT.md +Ограничения / известные ошибки: нет отдельного S3-пользователя и нет сборщика объектов, оставшихся после сбоя между PUT и COMMIT +Источники и использование ИИ: RustFS, библиотека minio, помощь ИИ в Cursor -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..476acdd9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.12.6-slim-bookworm + +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app + +WORKDIR /app +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY --chown=app:app src/ /app/ +COPY --chown=app:app migrations/ /app/migrations/ + +USER app + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab + +EXPOSE 8080 + +ARG BUILD_ID=dev +LABEL lab.build-id=$BUILD_ID + +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=10s --timeout=3s --start-period=8s --retries=5 \ + CMD python3 -c "import urllib.request,sys; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2); sys.exit(0)" + +CMD ["python3", "/app/server.py"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..3fd6a42d 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,68 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Самсонов Александр, БВТ2303 +- Целевая оценка: 3 +- Commit/PR: локально, до публикации +- Источники и помощь ИИ: контракт лабораторной, RustFS 1.0.4 как S3-совместимое хранилище (образы MinIO на Docker Hub к ноябрю 2026 недоступны), библиотека minio 7.2.15, перенос стенда lab3. Сборку сервера и проверки помог сделать ИИ в Cursor. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine с Compose v2 и Python 3.9+ для публичных тестов. Образ API — Python 3.12.6. Проверка выполнена в WSL Ubuntu 22.04: Docker 29.1.3, Compose 2.40.3. -## Архитектура и решения +```sh +cp .env.example .env +chmod +x scripts/up.sh scripts/restart.sh scripts/stop-one.sh scripts/restore-api.sh scripts/check-private-bucket.sh +docker compose --env-file .env -f compose.yaml up -d --build --scale api=2 --remove-orphans +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Тот же запуск: `sh scripts/up.sh`. Дождитесь `GET http://127.0.0.1:8080/health/ready` → `{"status":"ok"}`. -## Доказательства критериев +Публичный порт только у `lb`: `127.0.0.1:8080`. У API и RustFS порты наружу не опубликованы. Миграции выполняет одноразовый `migrate`, приватный bucket создаёт одноразовый `s3-init` (`mc` не нужен: клиент S3 вызывает `make_bucket`, если bucket ещё нет). Повторный запуск bucket не публикует. Политика anonymous не ставится. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +Остановка без удаления данных: + +```sh +docker compose --env-file .env -f compose.yaml stop +``` + +Удаление учебных volumes: + +```sh +docker compose --env-file .env -f compose.yaml down -v +``` + +После `scripts/stop-one.sh` остановленная реплика не поднимается сама. Вернуть её: `sh scripts/restore-api.sh`. + +Образы: `python:3.12.6-slim-bookworm`, `postgres:16.6-bookworm`, `valkey/valkey:8.0.2`, `traefik:v3.7.16`, `rustfs/rustfs:1.0.4`. + +## Архитектура + +``` +Клиент → Traefik → API ×2 → PostgreSQL (пользователи, чаты, сообщения, метаданные вложений) + → Valkey (сессии, AOF) + → RustFS (байты оригинала, volume s3data) +``` + +Из lab3 сохранены две реплики, общий PostgreSQL и Valkey, Traefik без sticky, `X-Instance-Id`, идемпотентность POST сообщения, drain по SIGTERM. API не хранит оригинал на своём диске: ключ объекта `o/` назначает сервер, `filename` только в метаданных. + +`/health/ready` отвечает 200, только если живы PostgreSQL, Valkey и bucket. Иначе 503, и Traefik перестаёт слать трафик на реплику. Период проверки Traefik — 1 с, timeout — 1 с, верхняя оценка исключения мёртвого backend — 2 с. + +201 на загрузку возвращается после PUT в bucket и COMMIT строки `attachments`. Если запись в БД не прошла, процесс удаляет только что положенный объект. Если процесс умрёт между PUT и COMMIT, объект может остаться без строки в PostgreSQL: фонового сборщика мусора нет. + +Лимит байт проверяется по `Content-Length` до чтения тела. Размеры PNG/JPEG читаются из IHDR/SOF, пиксели не распаковываются. Больше 10 MiB или 40 млн пикселей — 413. Не PNG/JPEG или несовпадение байтов и `Content-Type` — 415. Имя с `/`, `\` или управляющим символом — 400. + +## Доказательства + +| Критерий | Команда | Результат | Файл | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke, checksum, чужой пользователь | `python3 tests/smoke.py --base-url http://127.0.0.1:8080` | PASS | evidence/smoke.txt | +| Restart API, PostgreSQL, Valkey и RustFS, тот же токен и тот же файл | `python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080` | PASS | evidence/persistence.txt | +| Upload, остановка обслужившей реплики, чтение через другую | `python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080` | 2 реплики, 139 успешных записей, 4 ошибки, первое восстановление 3.63 с, PASS | evidence/failover.txt | +| Опасное имя, подмена MIME, 50 млн пикселей, чужой чат | `python3 scripts/grade3_extra.py --base-url http://127.0.0.1:8080` | PASS, коды 400/415/413/404 | evidence/grade3-extra.txt | +| Content-Length 10485761 при теле в 1 байт, прямо в API | `docker compose cp scripts/check_size_header.py api:/tmp/check_size_header.py` и `exec api python3 /tmp/check_size_header.py` | 413 за 0.001 с | evidence/grade3-extra.txt | +| Приватный bucket | `sh scripts/check-private-bucket.sh` | анонимный GET 403 | evidence/grade3-extra.txt | +| Недоступность S3 | `docker compose stop s3`, затем запрос `/health/ready` | 503; после `start s3` снова 200 | evidence/s3-down.txt | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Ограничения -## Эксперименты и отказы - -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | - -## Ограничения и следующие шаги - -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Уровень 4 не делался: нет отдельного пользователя S3 с урезанными правами, нет фоновой сверки осиротевших объектов, нет замера RAM на 10 одновременных загрузках. Консоль RustFS выключена (`RUSTFS_CONSOLE_ENABLE=false`) и наружу не опубликована. Один и тот же ключ из `.env` является корневым ключом RustFS и ключом API. Это учебный локальный стенд. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..d76c3f7b --- /dev/null +++ b/compose.yaml @@ -0,0 +1,209 @@ +# Локальный стенд lab4: клиент → Traefik → API ×2 → PostgreSQL + Valkey + RustFS. +# Публичный порт только у lb. У API и S3 нет ports и нет container_name. +# Миграции и создание bucket — одноразовые сервисы, не каждая реплика. +name: mtusi-lab4 + +services: + postgres: + image: postgres:16.6-bookworm + environment: + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} + POSTGRES_DB: ${POSTGRES_DB:-messenger} + volumes: + - pgdata:/var/lib/postgresql/data + shm_size: 128mb + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-messenger} -d ${POSTGRES_DB:-messenger}"] + interval: 2s + timeout: 3s + retries: 20 + start_period: 5s + restart: unless-stopped + stop_grace_period: 20s + networks: [lab] + deploy: + resources: + limits: + cpus: "0.75" + memory: 512M + + valkey: + image: valkey/valkey:8.0.2 + command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", "--save", "60", "1"] + volumes: + - valkeydata:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 2s + timeout: 3s + retries: 20 + start_period: 3s + restart: unless-stopped + stop_grace_period: 10s + networks: [lab] + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + s3: + image: rustfs/rustfs:1.0.4 + environment: + RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env} + RUSTFS_SECRET_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env} + RUSTFS_ADDRESS: ":9000" + RUSTFS_CONSOLE_ENABLE: "false" + RUSTFS_VOLUMES: /data + RUSTFS_CHECK_UPDATE: "false" + volumes: + - s3data:/data + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:9000/health"] + interval: 2s + timeout: 3s + retries: 20 + start_period: 5s + restart: unless-stopped + stop_grace_period: 15s + networks: [lab] + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + + migrate: + build: . + image: mtusi-messenger:lab4 + command: ["python3", "/app/migrate.py"] + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-messenger} + depends_on: + postgres: + condition: service_healthy + restart: "no" + networks: [lab] + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + + s3-init: + image: mtusi-messenger:lab4 + command: ["python3", "/app/s3_init.py"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + depends_on: + s3: + condition: service_healthy + restart: "no" + networks: [lab] + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + api: + image: mtusi-messenger:lab4 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: valkey + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DRAIN_SECONDS: ${DRAIN_SECONDS:-3} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-messenger} + REDIS_URL: redis://valkey:6379/0 + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + valkey: + condition: service_healthy + expose: + - "8080" + labels: + - traefik.enable=true + - traefik.docker.network=mtusi-lab4-net + - traefik.http.routers.messenger.rule=PathPrefix(`/`) + - traefik.http.routers.messenger.entrypoints=web + - traefik.http.services.messenger.loadbalancer.server.port=8080 + - traefik.http.services.messenger.loadbalancer.healthcheck.path=/health/ready + - traefik.http.services.messenger.loadbalancer.healthcheck.interval=1s + - traefik.http.services.messenger.loadbalancer.healthcheck.timeout=1s + - traefik.http.services.messenger.loadbalancer.healthcheck.scheme=http + - traefik.http.services.messenger.loadbalancer.passhostheader=true + healthcheck: + test: + - CMD + - python3 + - -c + - import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2) + interval: 2s + timeout: 2s + retries: 8 + start_period: 8s + init: true + restart: unless-stopped + stop_grace_period: 10s + stop_signal: SIGTERM + networks: [lab] + deploy: + replicas: 2 + resources: + limits: + cpus: "0.50" + memory: 512M + + lb: + image: traefik:v3.7.16 + command: ["--configFile=/etc/traefik/traefik.yml"] + volumes: + - ./infra/traefik.yml:/etc/traefik/traefik.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + ports: + - "127.0.0.1:8080:8080" + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8082/ping"] + interval: 2s + timeout: 2s + retries: 8 + start_period: 3s + restart: unless-stopped + stop_grace_period: 5s + networks: [lab] + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + +networks: + lab: + name: mtusi-lab4-net + driver: bridge + +volumes: + pgdata: + valkeydata: + s3data: diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..ab3be025 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 139, "failed_attempts": 4, "first_recovery_seconds": 3.63} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/grade3-extra.txt b/evidence/grade3-extra.txt new file mode 100644 index 00000000..09644eb4 --- /dev/null +++ b/evidence/grade3-extra.txt @@ -0,0 +1,3 @@ +PASS: filename, MIME, pixels, outsider, foreign chat +oversized Content-Length -> 413 in 0.001s +anonymous S3 status 403 diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/s3-down.txt b/evidence/s3-down.txt new file mode 100644 index 00000000..840b51fb --- /dev/null +++ b/evidence/s3-down.txt @@ -0,0 +1,4 @@ +docker compose stop s3 +HTTP/1.1 503 Service Unavailable +После docker compose start s3 готовность /health/ready снова 200. +Анонимный GET http://s3:9000/messenger-attachments/ из сети стенда: 403. diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..c554fd6f --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/infra/traefik.yml b/infra/traefik.yml new file mode 100644 index 00000000..a6064a75 --- /dev/null +++ b/infra/traefik.yml @@ -0,0 +1,45 @@ +# Traefik 3.7.16. Без sticky cookie. +# Адреса API Traefik берёт из Docker и обновляет по событиям start/die, +# поэтому новая реплика и остановленная не зависят от кэша одного DNS-ответа. +# Healthcheck: период 1s, timeout 1s, одна неуспешная проверка исключает backend. +# Верхняя оценка обнаружения отказа: 1s до следующей проверки + 1s timeout = 2s. +# Таймауты чтения запроса и ответа — 60s, чтобы прошёл оригинал до 10 MiB. +# Обнаружение мёртвой реплики от этого не зависит: healthcheck по-прежнему 1s+1s. + +global: + checkNewVersion: false + sendAnonymousUsage: false + +entryPoints: + web: + address: ":8080" + http: + aliasHeadersStrategy: reject + transport: + respondingTimeouts: + readTimeout: 60s + writeTimeout: 60s + idleTimeout: 30s + ping: + address: "127.0.0.1:8082" + http: + aliasHeadersStrategy: delete + +ping: + entryPoint: ping + +providers: + docker: + endpoint: unix:///var/run/docker.sock + exposedByDefault: false + network: mtusi-lab4-net + watch: true + +serversTransport: + forwardingTimeouts: + dialTimeout: 1s + responseHeaderTimeout: 60s + idleConnTimeout: 1s + +log: + level: INFO diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..3c2cc42f --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,49 @@ +-- Пользователи, чаты, участники, сообщения. +-- Сессии здесь не хранятся: их источник истины — Valkey (AOF + volume). +-- Идемпотентность отправки — в 002_idempotency.sql. +-- +-- Транзакции приложения: +-- регистрация — один INSERT, уникальность username держит UNIQUE +-- создание чата — одна транзакция: чат и все chat_members, иначе откат +-- сообщение и запись идемпотентности — одна транзакция +-- +-- Индекс истории messages_history_idx покрывает keyset-запрос +-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +-- ORDER BY created_at, id +-- LIMIT $4 + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT users_username_unique UNIQUE (username) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE INDEX chats_created_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position integer NOT NULL, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..bab92309 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,18 @@ +-- Общий результат POST сообщения. +-- Область: user_id + http_method + path + idem_key. +-- Строка пишется в той же транзакции, что и messages, поэтому 5xx +-- до COMMIT не оставляет ни сообщения, ни «окончательного» ключа. +-- Повтор с другим request_hash в пределах TTL — отказ без второй вставки. + +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + http_method text NOT NULL, + path text NOT NULL, + idem_key text NOT NULL, + request_hash text NOT NULL, + message_id uuid NOT NULL REFERENCES messages (id), + created_at timestamptz NOT NULL, + PRIMARY KEY (user_id, http_method, path, idem_key) +); + +CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..c21e8dcb --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,27 @@ +-- Метаданные вложения. Байты лежат в приватном S3-bucket, не в этой таблице. +-- object_key назначает сервер. filename только для отображения. +-- Связь сообщения и вложения — message_attachments, порядок задаёт position. + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT attachments_object_key_unique UNIQUE (object_key) +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + attachment_id uuid NOT NULL REFERENCES attachments (id), + position integer NOT NULL, + PRIMARY KEY (message_id, attachment_id) +); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..509f3b89 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +psycopg[binary]==3.2.3 +psycopg-pool==3.2.3 +redis==5.1.1 +minio==7.2.15 diff --git a/scripts/check-private-bucket.sh b/scripts/check-private-bucket.sh new file mode 100755 index 00000000..b405b56c --- /dev/null +++ b/scripts/check-private-bucket.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env -f compose.yaml exec -T api python3 - <<'PY' +from urllib.request import urlopen +try: + urlopen("http://s3:9000/messenger-attachments/", timeout=3) + raise SystemExit("anonymous bucket read was allowed") +except Exception as exc: + code = getattr(exc, "code", None) + if code not in (401, 403): + raise SystemExit("unexpected result %s %s" % (type(exc).__name__, code)) + print("anonymous S3 status", code) +PY diff --git a/scripts/check_size_header.py b/scripts/check_size_header.py new file mode 100644 index 00000000..00159bda --- /dev/null +++ b/scripts/check_size_header.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Проверка лимита размера на самом API, мимо балансировщика. Токен не печатает.""" +import http.client +import importlib +import json +import time +import uuid +import urllib.request + +urlerr = importlib.import_module("urllib.error") + +BASE = "http://127.0.0.1:8080" + + +def call(method, path, body=None, headers=None, timeout=5): + req = urllib.request.Request(BASE + path, data=body, headers=headers or {}, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + return response.status, response.read() + except urlerr.HTTPError as exc: + return exc.code, exc.read() + + +def main(): + username = "u_sz_" + uuid.uuid4().hex[:12] + password = "T3st-" + uuid.uuid4().hex + status, raw = call( + "POST", "/api/v1/users", + json.dumps({"username": username, "display_name": "Size", "password": password}).encode(), + {"Content-Type": "application/json"}, + ) + if status != 201: + raise SystemExit("register %s" % status) + user = json.loads(raw) + import base64 + token = base64.b64encode((username + ":" + password).encode()).decode() + status, raw = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + token}) + if status != 201: + raise SystemExit("login %s" % status) + bearer = json.loads(raw)["token"] + auth = {"Authorization": "Bearer " + bearer, "Content-Type": "application/json"} + status, raw = call( + "POST", "/api/v1/chats", + json.dumps({"title": "limit", "member_ids": [user["id"]]}).encode(), + auth, + ) + if status != 201: + raise SystemExit("chat %s" % status) + chat = json.loads(raw)["id"] + # Заголовок больше лимита, тело из одного байта: API должен ответить до чтения остальных байт. + started = time.monotonic() + conn = http.client.HTTPConnection("127.0.0.1", 8080, timeout=3) + conn.request( + "POST", + "/api/v1/chats/" + chat + "/attachments?filename=big.png", + body=b"x", + headers={ + "Authorization": "Bearer " + bearer, + "Content-Type": "image/png", + "Content-Length": "10485761", + }, + ) + response = conn.getresponse() + status = response.status + elapsed = time.monotonic() - started + conn.close() + if status != 413: + raise SystemExit("expected 413, got %s" % status) + if elapsed > 2: + raise SystemExit("limit check took too long: %.3f" % elapsed) + print("oversized Content-Length -> %s in %.3fs" % (status, elapsed)) + + +if __name__ == "__main__": + main() diff --git a/scripts/grade3_extra.py b/scripts/grade3_extra.py new file mode 100644 index 00000000..3fb79f1c --- /dev/null +++ b/scripts/grade3_extra.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Негативные проверки вложений для уровня 3. Токены не печатает.""" +import hashlib +import struct +import sys +import uuid +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 + + +def png(width: int, height: int) -> bytes: + signature = b"\x89PNG\r\n\x1a\n" + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + chunk = b"IHDR" + ihdr + crc = zlib.crc32(chunk) & 0xFFFFFFFF + return signature + struct.pack(">I", len(ihdr)) + chunk + struct.pack(">I", crc) + + +def main(): + args = arguments("Lab4 grade-3 attachment checks") + base = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = base.bootstrap() + other = base.call( + "POST", "/api/v1/chats", 201, actor=alice, + body={"title": "Другой чат", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + sample = (ROOT / "tests/fixtures/sample.png").read_bytes() + att, content = base.attachment(alice, chat) + linked = base.message(alice, chat, "", attachments=[att["id"]])["json"] + check(linked["attachment_ids"] == [att["id"]], "message lost attachment") + downloaded = base.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob) + check(downloaded["bytes"] == content, "bob download differs") + check(hashlib.sha256(downloaded["bytes"]).hexdigest() == att["sha256"], "checksum differs") + for suffix in ("", "/content"): + base.call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, actor=eve) + base.call( + "POST", "/api/v1/chats/" + other["id"] + "/messages", 404, actor=alice, + body={"text": "", "attachment_ids": [att["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + base.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=a/b.png", 400, + actor=alice, raw=sample, headers={"Content-Type": "image/png"}, + ) + base.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=photo.png", 415, + actor=alice, raw=sample, headers={"Content-Type": "image/jpeg"}, + ) + base.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=huge.png", 413, + actor=alice, raw=png(10000, 5000), headers={"Content-Type": "image/png"}, + ) + print("PASS: filename, MIME, pixels, outsider, foreign chat") + + +if __name__ == "__main__": + run(main) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..7f2f6c7a --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Перезапуск API и всех хранилищ. Volumes не удаляются, схема и объекты остаются. +docker compose --env-file .env -f compose.yaml restart api postgres valkey s3 diff --git a/scripts/restore-api.sh b/scripts/restore-api.sh new file mode 100755 index 00000000..902ae572 --- /dev/null +++ b/scripts/restore-api.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +replicas=$(sed -n 's/^API_REPLICAS=//p' .env | tr -d '\r') +if [ -z "${replicas}" ]; then + replicas=2 +fi +docker compose --env-file .env -f compose.yaml up -d --no-deps --scale "api=${replicas}" --no-recreate api diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..1bfd2812 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" +cd "$(dirname "$0")/.." + +found="" +for cid in $(docker compose --env-file .env -f compose.yaml ps -q api); do + host=$(docker inspect -f '{{.Config.Hostname}}' "$cid") + short=$(printf '%s' "$cid" | cut -c1-12) + if [ "$host" = "$TARGET_INSTANCE_ID" ] || [ "$short" = "$TARGET_INSTANCE_ID" ]; then + found=$cid + break + fi +done + +if [ -z "$found" ]; then + echo "API replica was not found" >&2 + exit 1 +fi + +# docker stop + restart: unless-stopped оставляет реплику выключенной. +docker stop -t 8 "$found" >/dev/null diff --git a/scripts/up.sh b/scripts/up.sh new file mode 100755 index 00000000..5d111aa2 --- /dev/null +++ b/scripts/up.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +if [ ! -f .env ]; then + echo "Copy .env.example to .env" >&2 + exit 1 +fi +replicas=$(sed -n 's/^API_REPLICAS=//p' .env | tr -d '\r') +if [ -z "${replicas}" ]; then + replicas=2 +fi +docker compose --env-file .env -f compose.yaml up -d --build --scale "api=${replicas}" --remove-orphans diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..c975f06b --- /dev/null +++ b/src/db.py @@ -0,0 +1,603 @@ +"""PostgreSQL: пользователи, чаты, участники, сообщения, идемпотентность.""" +from __future__ import annotations + +import hashlib +import os +import time +import uuid +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path + +import psycopg +from psycopg.errors import UniqueViolation +from psycopg_pool import ConnectionPool, PoolTimeout + +from errors import ApiError + +_pool: ConnectionPool | None = None + + +def migrations_dir() -> Path: + here = Path(__file__).resolve().parent + for candidate in (here / "migrations", here.parent / "migrations"): + if candidate.is_dir(): + return candidate + return here.parent / "migrations" + + +def database_url() -> str: + path = os.environ.get("DATABASE_URL_FILE", "").strip() + if path and Path(path).is_file(): + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get("DATABASE_URL", "").strip() + + +def idempotency_ttl() -> int: + raw = os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400") + try: + value = int(raw) + except ValueError: + return 86400 + return value if value >= 1 else 86400 + + +def init() -> None: + global _pool + url = database_url() + if not url: + raise RuntimeError("database is not configured") + _pool = ConnectionPool( + url, + min_size=0, + max_size=12, + timeout=3, + open=True, + kwargs={ + "connect_timeout": 3, + "autocommit": False, + "options": "-c statement_timeout=5000", + }, + ) + + +def migrate_with_retry() -> None: + last = None + for _ in range(30): + try: + migrate() + return + except Exception as exc: + last = exc + time.sleep(1) + raise RuntimeError("migrations failed") from last + + +def migrate() -> None: + url = database_url() + if not url: + raise RuntimeError("database is not configured") + with psycopg.connect(url, connect_timeout=3, autocommit=False) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (842021,)) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + conn.commit() + applied = { + row[0] + for row in conn.execute("SELECT version FROM schema_migrations").fetchall() + } + for path in sorted(migrations_dir().glob("*.sql")): + if path.name in applied: + continue + sql = path.read_text(encoding="utf-8") + with conn.transaction(): + for statement in sql.split(";"): + if statement.strip(): + conn.execute(statement) + conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (path.name,), + ) + + +@contextmanager +def connection(): + if _pool is None: + raise ApiError(503, "unavailable", "База данных недоступна") + try: + with _pool.connection() as conn: + yield conn + except ApiError: + raise + except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError) as exc: + raise ApiError(503, "unavailable", "База данных недоступна") from exc + + +def ping() -> bool: + try: + with connection() as conn: + conn.execute("SELECT 1") + return True + except ApiError: + return False + + +def close() -> None: + if _pool is not None: + _pool.close() + + +def user_json(row) -> dict: + return { + "id": str(row[0]), + "username": row[1], + "display_name": row[2], + "created_at": format_ts(row[3]), + } + + +def format_ts(value: datetime) -> str: + value = value.astimezone(timezone.utc) + return value.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def create_user(username: str, display_name: str, password_hash: str, created: datetime) -> dict: + uid = uuid.uuid4() + try: + with connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (uid, username, display_name, password_hash, created), + ).fetchone() + except UniqueViolation as exc: + raise ApiError(409, "conflict", "username занят") from exc + return user_json(row) + + +def user_for_login(username: str): + with connection() as conn: + return conn.execute( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users + WHERE username = %s + """, + (username,), + ).fetchone() + + +def get_user(user_id: str) -> dict | None: + with connection() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, created_at + FROM users + WHERE id = %s + """, + (uuid.UUID(user_id),), + ).fetchone() + if row is None: + return None + return user_json(row) + + +def create_chat(title: str, member_ids: list[str], created: datetime) -> dict: + cid = uuid.uuid4() + members = [uuid.UUID(item) for item in member_ids] + with connection() as conn: + with conn.transaction(): + conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (cid, title, created), + ) + found = conn.execute( + "SELECT id FROM users WHERE id = ANY(%s::uuid[])", + (members,), + ).fetchall() + if {row[0] for row in found} != set(members): + raise ApiError(400, "invalid_request", "неизвестный участник") + for position, member in enumerate(members): + conn.execute( + """ + INSERT INTO chat_members (chat_id, user_id, position) + VALUES (%s, %s, %s) + """, + (cid, member, position), + ) + return { + "id": str(cid), + "title": title, + "member_ids": list(member_ids), + "created_at": format_ts(created), + } + + +def _chat_row(row) -> dict: + return { + "id": str(row[0]), + "title": row[1], + "created_at": format_ts(row[2]), + "member_ids": [str(item) for item in row[3]], + } + + +def get_chat(chat_id: str, user_id: str) -> dict: + with connection() as conn: + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at, + array_agg(cm.user_id ORDER BY cm.position) + FROM chats c + JOIN chat_members mine + ON mine.chat_id = c.id AND mine.user_id = %s + JOIN chat_members cm ON cm.chat_id = c.id + WHERE c.id = %s + GROUP BY c.id + """, + (uuid.UUID(user_id), uuid.UUID(chat_id)), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Чат не найден") + chat = _chat_row(row) + return { + "id": chat["id"], + "title": chat["title"], + "member_ids": chat["member_ids"], + "created_at": chat["created_at"], + } + + +def list_chats(user_id: str, limit: int, cursor) -> dict: + params: list = [uuid.UUID(user_id)] + where = "" + if cursor is not None: + created, chat_id = cursor + where = "WHERE (c.created_at, c.id) > (%s, %s)" + params.extend([created, uuid.UUID(chat_id)]) + params.append(limit + 1) + sql = f""" + SELECT c.id, c.title, c.created_at, + array_agg(cm.user_id ORDER BY cm.position) + FROM chats c + JOIN chat_members mine + ON mine.chat_id = c.id AND mine.user_id = %s + JOIN chat_members cm ON cm.chat_id = c.id + {where} + GROUP BY c.id + ORDER BY c.created_at, c.id + LIMIT %s + """ + with connection() as conn: + rows = conn.execute(sql, params).fetchall() + page = rows[:limit] + next_cursor = None + if len(rows) > limit and page: + last = page[-1] + next_cursor = (last[2], str(last[0])) + return { + "items": [ + { + "id": str(row[0]), + "title": row[1], + "member_ids": [str(item) for item in row[3]], + "created_at": format_ts(row[2]), + } + for row in page + ], + "next_cursor_parts": next_cursor, + } + + +def _lock_key(scope: str) -> int: + digest = hashlib.sha256(scope.encode("utf-8")).digest() + return int.from_bytes(digest[:8], "big", signed=True) + + +def _is_member(conn, chat_id: str, user_id: str) -> bool: + row = conn.execute( + """ + SELECT 1 FROM chat_members + WHERE chat_id = %s AND user_id = %s + """, + (uuid.UUID(chat_id), uuid.UUID(user_id)), + ).fetchone() + return row is not None + + +def _attachment_ids(value) -> list: + if not value: + return [] + return [str(item) for item in value] + + +def _load_message(conn, message_id) -> dict: + row = conn.execute( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + COALESCE( + array_agg(ma.attachment_id ORDER BY ma.position) + FILTER (WHERE ma.attachment_id IS NOT NULL), + '{}'::uuid[] + ) + FROM messages m + LEFT JOIN message_attachments ma ON ma.message_id = m.id + WHERE m.id = %s + GROUP BY m.id + """, + (message_id,), + ).fetchone() + if row is None: + raise ApiError(500, "internal_error", "Внутренняя ошибка") + return message_json(row) + + +def _require_chat_attachments(conn, chat_id: str, attachment_ids: list[str]) -> None: + if not attachment_ids: + return + rows = conn.execute( + """ + SELECT id::text, chat_id::text + FROM attachments + WHERE id = ANY(%s::uuid[]) + """, + (attachment_ids,), + ).fetchall() + found = {row[0]: row[1] for row in rows} + for att_id in attachment_ids: + if found.get(att_id) != chat_id: + raise ApiError(404, "not_found", "Вложение не найдено") + + +def create_message_idempotent( + chat_id: str, + sender_id: str, + text: str, + attachment_ids: list[str], + created: datetime, + method: str, + path: str, + idem_key: str, + request_hash: str, +) -> dict: + """Один ключ — одно сообщение, в том числе при гонке двух реплик. + + pg_advisory_xact_lock сериализует одинаковую область. + INSERT сообщения, связей и ключа коммитятся вместе. + """ + scope = "%s|%s|%s|%s" % (sender_id, method, path, idem_key) + ttl = idempotency_ttl() + with connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s::bigint)", (_lock_key(scope),)) + existing = conn.execute( + """ + SELECT request_hash, message_id, created_at + FROM idempotency_keys + WHERE user_id = %s AND http_method = %s AND path = %s AND idem_key = %s + """, + (uuid.UUID(sender_id), method, path, idem_key), + ).fetchone() + if existing is not None: + age = conn.execute("SELECT now()").fetchone()[0] - existing[2] + if age.total_seconds() > ttl: + conn.execute( + """ + DELETE FROM idempotency_keys + WHERE user_id = %s AND http_method = %s AND path = %s AND idem_key = %s + """, + (uuid.UUID(sender_id), method, path, idem_key), + ) + existing = None + if existing is not None: + if existing[0] != request_hash: + raise ApiError(409, "conflict", "повторный ключ с другим сообщением") + if not _is_member(conn, chat_id, sender_id): + raise ApiError(404, "not_found", "Чат не найден") + return _load_message(conn, existing[1]) + if not _is_member(conn, chat_id, sender_id): + raise ApiError(404, "not_found", "Чат не найден") + _require_chat_attachments(conn, chat_id, attachment_ids) + mid = uuid.uuid4() + conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + (mid, uuid.UUID(chat_id), uuid.UUID(sender_id), text, created), + ) + for position, att_id in enumerate(attachment_ids): + conn.execute( + """ + INSERT INTO message_attachments (message_id, attachment_id, position) + VALUES (%s, %s, %s) + """, + (mid, uuid.UUID(att_id), position), + ) + conn.execute( + """ + INSERT INTO idempotency_keys ( + user_id, http_method, path, idem_key, request_hash, message_id, created_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s) + """, + (uuid.UUID(sender_id), method, path, idem_key, request_hash, mid, created), + ) + return _load_message(conn, mid) + + +def list_messages(chat_id: str, user_id: str, limit: int, cursor) -> dict: + params: list = [uuid.UUID(chat_id), uuid.UUID(chat_id), uuid.UUID(user_id)] + cursor_sql = "" + if cursor is not None: + created, message_id = cursor + cursor_sql = "AND (m.created_at, m.id) > (%s, %s)" + params.extend([created, uuid.UUID(message_id)]) + params.append(limit + 1) + sql = f""" + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + COALESCE( + array_agg(ma.attachment_id ORDER BY ma.position) + FILTER (WHERE ma.attachment_id IS NOT NULL), + '{{}}'::uuid[] + ) + FROM messages m + LEFT JOIN message_attachments ma ON ma.message_id = m.id + WHERE m.chat_id = %s + AND EXISTS ( + SELECT 1 FROM chat_members + WHERE chat_id = %s AND user_id = %s + ) + {cursor_sql} + GROUP BY m.id + ORDER BY m.created_at, m.id + LIMIT %s + """ + with connection() as conn: + rows = conn.execute(sql, params).fetchall() + if not rows: + with connection() as conn: + member = conn.execute( + """ + SELECT 1 FROM chat_members + WHERE chat_id = %s AND user_id = %s + """, + (uuid.UUID(chat_id), uuid.UUID(user_id)), + ).fetchone() + if member is None: + raise ApiError(404, "not_found", "Чат не найден") + page = rows[:limit] + next_cursor = None + if len(rows) > limit and page: + last = page[-1] + next_cursor = (last[4], str(last[0])) + return { + "items": [message_json(row) for row in page], + "next_cursor_parts": next_cursor, + } + + +def message_json(row) -> dict: + return { + "id": str(row[0]), + "chat_id": str(row[1]), + "sender_id": str(row[2]), + "text": row[3], + "created_at": format_ts(row[4]), + "attachment_ids": _attachment_ids(row[5]), + } + + +def is_member(chat_id: str, user_id: str) -> bool: + try: + with connection() as conn: + return _is_member(conn, chat_id, user_id) + except ApiError: + raise + + +def insert_attachment( + att_id: str, + chat_id: str, + owner_id: str, + filename: str, + content_type: str, + size_bytes: int, + digest: str, + object_key: str, + width: int, + height: int, + created: datetime, +) -> dict: + with connection() as conn: + with conn.transaction(): + if not _is_member(conn, chat_id, owner_id): + raise ApiError(404, "not_found", "Чат не найден") + row = conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, width, height, created_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, object_key, created_at + """, + ( + uuid.UUID(att_id), + uuid.UUID(chat_id), + uuid.UUID(owner_id), + filename, + content_type, + size_bytes, + digest, + object_key, + width, + height, + created, + ), + ).fetchone() + return _attachment_public(row) + + +def get_attachment(att_id: str, user_id: str) -> dict: + row = _attachment_row(att_id, user_id) + return _attachment_public(row) + + +def get_attachment_record(att_id: str, user_id: str) -> dict: + row = _attachment_row(att_id, user_id) + return { + "content_type": row[4], + "size_bytes": row[5], + "sha256": row[6], + "object_key": row[7], + } + + +def _attachment_row(att_id: str, user_id: str): + with connection() as conn: + row = conn.execute( + """ + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, + a.size_bytes, a.sha256, a.object_key, a.created_at + FROM attachments a + WHERE a.id = %s + AND EXISTS ( + SELECT 1 FROM chat_members + WHERE chat_id = a.chat_id AND user_id = %s + ) + """, + (uuid.UUID(att_id), uuid.UUID(user_id)), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Вложение не найдено") + return row + + +def _attachment_public(row) -> dict: + created = format_ts(row[8]) + return { + "id": str(row[0]), + "chat_id": str(row[1]), + "owner_id": str(row[2]), + "filename": row[3], + "content_type": row[4], + "size_bytes": row[5], + "sha256": row[6], + "status": "ready", + "variants": [], + "error_code": None, + "created_at": created, + "updated_at": created, + } diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..595498ed --- /dev/null +++ b/src/errors.py @@ -0,0 +1,8 @@ +"""Ошибки API. Текст не содержит SQL, паролей и токенов.""" + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str): + self.status = status + self.code = code + self.message = message diff --git a/src/images.py b/src/images.py new file mode 100644 index 00000000..2abbb177 --- /dev/null +++ b/src/images.py @@ -0,0 +1,74 @@ +"""Размеры PNG/JPEG по заголовкам, без распаковки пикселей.""" +from __future__ import annotations + +from errors import ApiError + +PNG_SIG = b"\x89PNG\r\n\x1a\n" +# SOF0..SOF3, SOF5..SOF7, SOF9..SOF11, SOF13..SOF15. Не DHT/JPG/DAC. +_JPEG_SOF = { + 0xC0, 0xC1, 0xC2, 0xC3, + 0xC5, 0xC6, 0xC7, + 0xC9, 0xCA, 0xCB, + 0xCD, 0xCE, 0xCF, +} + + +def sniff(data: bytes) -> str: + if data.startswith(PNG_SIG): + return "image/png" + if data.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + raise ApiError(415, "unsupported_media_type", "Файл не PNG и не JPEG") + + +def dimensions(data: bytes, content_type: str) -> tuple[int, int]: + if content_type == "image/png": + return _png_size(data) + if content_type == "image/jpeg": + return _jpeg_size(data) + raise ApiError(415, "unsupported_media_type", "Файл не PNG и не JPEG") + + +def _png_size(data: bytes) -> tuple[int, int]: + if len(data) < 24 or not data.startswith(PNG_SIG) or data[12:16] != b"IHDR": + raise ApiError(415, "unsupported_media_type", "Повреждённый PNG") + width = int.from_bytes(data[16:20], "big") + height = int.from_bytes(data[20:24], "big") + if width < 1 or height < 1: + raise ApiError(415, "unsupported_media_type", "Повреждённый PNG") + return width, height + + +def _jpeg_size(data: bytes) -> tuple[int, int]: + if len(data) < 4 or data[:2] != b"\xff\xd8": + raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG") + index = 2 + size = len(data) + while index < size: + if data[index] != 0xFF: + break + while index < size and data[index] == 0xFF: + index += 1 + if index >= size: + break + marker = data[index] + index += 1 + if marker in (0xD8, 0xD9) or marker == 0xDA: + break + if marker == 0x01 or 0xD0 <= marker <= 0xD7: + continue + if index + 2 > size: + break + segment = int.from_bytes(data[index:index + 2], "big") + if segment < 2 or index + segment > size: + raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG") + if marker in _JPEG_SOF: + if segment < 7: + raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG") + height = int.from_bytes(data[index + 3:index + 5], "big") + width = int.from_bytes(data[index + 5:index + 7], "big") + if width < 1 or height < 1: + raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG") + return width, height + index += segment + raise ApiError(415, "unsupported_media_type", "В JPEG нет размеров") diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..6ae5a613 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +"""Однократные миграции. Реплики API этот файл не запускают.""" +import sys + +import db + + +def main() -> None: + db.migrate_with_retry() + sys.stderr.write("migrations applied\n") + + +if __name__ == "__main__": + main() diff --git a/src/passwords.py b/src/passwords.py new file mode 100644 index 00000000..305eb35c --- /dev/null +++ b/src/passwords.py @@ -0,0 +1,43 @@ +"""Пароли: scrypt из hashlib. Не bcrypt — он режет пароль на 72 байтах.""" +from __future__ import annotations + +import base64 +import hashlib +import hmac +import secrets + +N = 2**14 +R = 8 +P = 1 +DKLEN = 32 + + +def hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + digest = hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=N, r=R, p=P, dklen=DKLEN, + maxmem=64 * 1024 * 1024) + return "scrypt$%d$%d$%d$%s$%s" % ( + N, R, P, + base64.b64encode(salt).decode("ascii"), + base64.b64encode(digest).decode("ascii"), + ) + + +def verify_password(password: str, stored: str) -> bool: + try: + algo, n, r, p, salt_b64, digest_b64 = stored.split("$") + if algo != "scrypt": + return False + salt = base64.b64decode(salt_b64) + expected = base64.b64decode(digest_b64) + actual = hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=int(n), r=int(r), p=int(p), + dklen=len(expected), maxmem=64 * 1024 * 1024) + except (ValueError, TypeError): + return False + return hmac.compare_digest(actual, expected) + + +# Одинаковая стоимость проверки, когда пользователя нет. +DUMMY_HASH = hash_password("dummy-password-not-used") diff --git a/src/s3_init.py b/src/s3_init.py new file mode 100644 index 00000000..c9ec2bda --- /dev/null +++ b/src/s3_init.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Однократное создание приватного bucket. Повторный запуск ничего не публикует.""" +import sys +import time + +import storage + + +def main() -> None: + storage.init() + last = None + for _ in range(30): + try: + storage.ensure_bucket() + sys.stderr.write("bucket ready\n") + return + except Exception as exc: + last = exc + time.sleep(1) + sys.stderr.write("bucket bootstrap failed\n") + raise SystemExit(1) from last + + +if __name__ == "__main__": + main() diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..493e41e4 --- /dev/null +++ b/src/server.py @@ -0,0 +1,701 @@ +#!/usr/bin/env python3 +"""Messenger lab4. Реплики за Traefik, метаданные в PostgreSQL, байты в S3.""" +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import re +import secrets +import signal +import socket +import socketserver +import sys +import threading +import time +import unicodedata +import uuid +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler +from urllib.parse import parse_qs, urlparse + +import db +import images +import passwords +import sessions +import storage +from errors import ApiError + +APP_HOST = os.environ.get("APP_HOST", "0.0.0.0") +APP_PORT = int(os.environ.get("APP_PORT", "8080")) +APP_ENV = os.environ.get("APP_ENV", "lab") +INSTANCE_ID = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname() + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}\Z") +UUID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\Z") +TIMESTAMP_RE = re.compile( + r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?Z\Z") +IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}\Z") +MAX_BODY = 1024 * 1024 + +_draining = threading.Event() +_inflight = 0 +_inflight_lock = threading.Lock() + + +def drain_seconds() -> float: + raw = os.environ.get("DRAIN_SECONDS", "3") + try: + value = float(raw) + except ValueError: + return 3.0 + if value < 0: + return 0.0 + if value > 6: + return 6.0 + return value + + +def now_utc() -> datetime: + current = datetime.now(timezone.utc) + return current.replace(microsecond=(current.microsecond // 1000) * 1000) + + +def is_blank(value: str) -> bool: + return value.strip() == "" + + +def require_object(body): + if not isinstance(body, dict): + raise ApiError(400, "invalid_request", "Тело должно быть JSON-объектом") + + +def reject_extra(body: dict, allowed: set): + extra = set(body) - allowed + if extra: + raise ApiError(400, "invalid_request", + "Лишние поля: " + ", ".join(sorted(extra))) + + +def parse_uuid(value, field: str) -> str: + if not isinstance(value, str) or not UUID_RE.match(value): + raise ApiError(400, "invalid_request", f"Некорректный UUID: {field}") + return value.lower() + + +def parse_display_name(value) -> str: + if not isinstance(value, str): + raise ApiError(400, "invalid_request", "display_name должен быть строкой") + if not (1 <= len(value) <= 100): + raise ApiError(400, "invalid_request", "display_name: 1..100 code points") + if is_blank(value): + raise ApiError(400, "invalid_request", "display_name не может быть пустым") + return value + + +def parse_username(value) -> str: + if not isinstance(value, str) or not USERNAME_RE.match(value): + raise ApiError(400, "invalid_request", "username: ^[a-z0-9_]{3,32}$") + return value + + +def parse_password(value) -> str: + if not isinstance(value, str): + raise ApiError(400, "invalid_request", "password должен быть строкой") + if not (12 <= len(value) <= 128): + raise ApiError(400, "invalid_request", "password: 12..128 code points") + if is_blank(value): + raise ApiError(400, "invalid_request", "password не может быть пустым") + return value + + +def parse_title(value) -> str: + if not isinstance(value, str): + raise ApiError(400, "invalid_request", "title должен быть строкой") + if not (1 <= len(value) <= 100): + raise ApiError(400, "invalid_request", "title: 1..100 code points") + if is_blank(value): + raise ApiError(400, "invalid_request", "title не может быть пустым") + return value + + +def max_upload_bytes() -> int: + raw = os.environ.get("MAX_UPLOAD_BYTES", "10485760") + try: + value = int(raw) + except ValueError: + return 10485760 + return value if value >= 1 else 10485760 + + +def max_image_pixels() -> int: + raw = os.environ.get("MAX_IMAGE_PIXELS", "40000000") + try: + value = int(raw) + except ValueError: + return 40000000 + return value if value >= 1 else 40000000 + + +def parse_text(value, has_attachments: bool) -> str: + if not isinstance(value, str): + raise ApiError(400, "invalid_request", "text должен быть строкой") + if len(value) > 4000: + raise ApiError(400, "invalid_request", "text: не больше 4000 code points") + if is_blank(value) and not has_attachments: + raise ApiError(400, "invalid_request", "text не может быть пустым без вложений") + return value + + +def parse_attachment_ids(value) -> list: + if value is None: + return [] + if not isinstance(value, list): + raise ApiError(400, "invalid_request", "attachment_ids должен быть массивом") + if len(value) > 10: + raise ApiError(400, "invalid_request", "attachment_ids: не больше 10") + seen = set() + result = [] + for item in value: + att_id = parse_uuid(item, "attachment_ids[]") + if att_id in seen: + raise ApiError(400, "invalid_request", "attachment_ids содержит дубликаты") + seen.add(att_id) + result.append(att_id) + return result + + +def parse_filename(value) -> str: + if not isinstance(value, str): + raise ApiError(400, "invalid_request", "filename: 1..255 code points") + if not (1 <= len(value) <= 255): + raise ApiError(400, "invalid_request", "filename: 1..255 code points") + for char in value: + if char in "/\\" or unicodedata.category(char) == "Cc": + raise ApiError(400, "invalid_request", "filename содержит запрещённый символ") + return value + + +def parse_member_ids(value) -> list: + if not isinstance(value, list): + raise ApiError(400, "invalid_request", "member_ids должен быть массивом") + if not (1 <= len(value) <= 100): + raise ApiError(400, "invalid_request", "member_ids: 1..100") + seen = set() + result = [] + for item in value: + uid = parse_uuid(item, "member_ids[]") + if uid in seen: + raise ApiError(400, "invalid_request", "member_ids содержит дубликаты") + seen.add(uid) + result.append(uid) + return result + + +def parse_limit(raw, default=50) -> int: + if raw is None: + return default + if isinstance(raw, str) and raw.strip() == "": + raise ApiError(400, "invalid_request", "limit должен быть целым") + try: + value = int(raw) + except (TypeError, ValueError): + raise ApiError(400, "invalid_request", "limit должен быть целым") + if not (1 <= value <= 100): + raise ApiError(400, "invalid_request", "limit: 1..100") + return value + + +def parse_optional_cursor(raw): + if raw is None: + return None + if not isinstance(raw, str) or raw.strip() == "": + raise ApiError(400, "invalid_request", "Некорректный cursor") + return raw + + +def parse_cursor_timestamp(value: str) -> datetime: + if not isinstance(value, str) or not TIMESTAMP_RE.match(value): + raise ApiError(400, "invalid_request", "Некорректный cursor") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ApiError(400, "invalid_request", "Некорректный cursor") from exc + return parsed + + +def parse_idempotency_key(headers) -> str: + raw = headers.get("Idempotency-Key") + if not isinstance(raw, str) or not IDEMPOTENCY_RE.match(raw): + raise ApiError(400, "invalid_request", "Некорректный Idempotency-Key") + return raw + + +def canonical_hash(text: str, attachment_ids: list) -> str: + # Порядок ключей JSON не важен. Отсутствующий attachment_ids равен []. + # Порядок элементов массива сохраняется: это часть тела запроса. + payload = {"attachment_ids": attachment_ids, "text": text} + raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def challenge_for(path: str) -> str: + if path == "/api/v1/auth/sessions": + return 'Basic realm="messenger", charset="UTF-8"' + return 'Bearer realm="messenger"' + + +def bearer_token(headers) -> str: + auth = headers.get("Authorization") or "" + parts = auth.split(None, 1) + if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1].strip(): + raise ApiError(401, "unauthorized", "Требуется Bearer") + return parts[1].strip() + + +def current_user(headers) -> dict: + token = bearer_token(headers) + user = db.get_user(sessions.user_id(token)) + if user is None: + raise ApiError(401, "unauthorized", "Сессия недействительна") + return user + + +def basic_credentials(headers) -> tuple[str, str]: + auth = headers.get("Authorization") or "" + parts = auth.split(None, 1) + if len(parts) != 2 or parts[0].lower() != "basic" or not parts[1].strip(): + raise ApiError(401, "unauthorized", "Требуется Basic") + try: + decoded = base64.b64decode(parts[1].strip(), validate=True).decode("utf-8") + except (ValueError, UnicodeDecodeError) as exc: + raise ApiError(401, "unauthorized", "Неверные учётные данные") from exc + if ":" not in decoded: + raise ApiError(401, "unauthorized", "Неверные учётные данные") + username, password = decoded.split(":", 1) + return username, password + + +def encode_cursor(chat_id: str, created_at: str, msg_id: str) -> str: + raw = f"{chat_id}|{created_at}|{msg_id}".encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor(cursor: str, chat_id: str): + pad = "=" * (-len(cursor) % 4) + try: + raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8") + except Exception as exc: + raise ApiError(400, "invalid_request", "Некорректный cursor") from exc + parts = raw.split("|") + if len(parts) != 3: + raise ApiError(400, "invalid_request", "Некорректный cursor") + c_chat, c_created, c_id = parts + if not UUID_RE.match(c_chat): + raise ApiError(400, "invalid_request", "Некорректный cursor") + if c_chat.lower() != chat_id: + raise ApiError(400, "invalid_request", "cursor от другого чата") + return parse_cursor_timestamp(c_created), parse_uuid(c_id, "cursor") + + +def encode_chat_cursor(created_at: str, chat_id: str) -> str: + raw = f"chat|{created_at}|{chat_id}".encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_chat_cursor(cursor: str): + pad = "=" * (-len(cursor) % 4) + try: + raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8") + except Exception as exc: + raise ApiError(400, "invalid_request", "Некорректный cursor") from exc + parts = raw.split("|") + if len(parts) != 3 or parts[0] != "chat": + raise ApiError(400, "invalid_request", "Некорректный cursor") + return parse_cursor_timestamp(parts[1]), parse_uuid(parts[2], "cursor") + + +def create_user(body) -> dict: + require_object(body) + reject_extra(body, {"username", "display_name", "password"}) + username = parse_username(body.get("username")) + display_name = parse_display_name(body.get("display_name")) + password = parse_password(body.get("password")) + return db.create_user(username, display_name, passwords.hash_password(password), now_utc()) + + +def create_session(headers) -> dict: + username, password = basic_credentials(headers) + row = None + try: + row = db.user_for_login(username) + except ApiError: + raise + stored = row[3] if row is not None else passwords.DUMMY_HASH + password_ok = passwords.verify_password(password, stored) + if row is None or not password_ok: + raise ApiError(401, "unauthorized", "Неверные учётные данные") + token = secrets.token_urlsafe(32) + ttl = sessions.session_ttl() + expires = now_utc() + timedelta(seconds=ttl) + sessions.create(token, str(row[0]), ttl) + return { + "token": token, + "token_type": "Bearer", + "expires_at": db.format_ts(expires), + "user": db.user_json((row[0], row[1], row[2], row[4])), + } + + +def logout(headers) -> None: + token = bearer_token(headers) + sessions.user_id(token) + sessions.revoke(token) + + +def create_chat(body, actor: dict) -> dict: + require_object(body) + reject_extra(body, {"title", "member_ids"}) + title = parse_title(body.get("title")) + member_ids = parse_member_ids(body.get("member_ids")) + if actor["id"] not in member_ids: + raise ApiError(400, "invalid_request", "вызывающий должен быть в member_ids") + return db.create_chat(title, member_ids, now_utc()) + + +def list_chats(actor: dict, limit: int, cursor) -> dict: + decoded = decode_chat_cursor(cursor) if cursor else None + page = db.list_chats(actor["id"], limit, decoded) + next_cursor = None + if page["next_cursor_parts"] is not None: + created, chat_id = page["next_cursor_parts"] + next_cursor = encode_chat_cursor(db.format_ts(created), chat_id) + return {"items": page["items"], "next_cursor": next_cursor} + + +def create_message(chat_id: str, body, actor: dict, idem_key: str) -> dict: + cid = parse_uuid(chat_id, "chat_id") + require_object(body) + reject_extra(body, {"text", "attachment_ids"}) + attachment_ids = parse_attachment_ids(body.get("attachment_ids")) + text = parse_text(body.get("text"), bool(attachment_ids)) + path = "/api/v1/chats/%s/messages" % cid + return db.create_message_idempotent( + cid, + actor["id"], + text, + attachment_ids, + now_utc(), + "POST", + path, + idem_key, + canonical_hash(text, attachment_ids), + ) + + +def upload_attachment(chat_id: str, actor: dict, filename: str, declared: str, data: bytes) -> dict: + detected = images.sniff(data) + if detected != declared: + raise ApiError(415, "unsupported_media_type", "Содержимое не совпадает с Content-Type") + width, height = images.dimensions(data, detected) + if width * height > max_image_pixels(): + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + att_id = str(uuid.uuid4()) + object_key = "o/" + att_id + digest = hashlib.sha256(data).hexdigest() + created = now_utc() + storage.put_bytes(object_key, data, detected) + try: + return db.insert_attachment( + att_id, chat_id, actor["id"], filename, detected, len(data), digest, + object_key, width, height, created, + ) + except Exception: + storage.delete_quiet(object_key) + raise + + +def read_attachment(attachment_id: str, actor: dict) -> dict: + return db.get_attachment(parse_uuid(attachment_id, "attachment_id"), actor["id"]) + + +def read_attachment_bytes(attachment_id: str, actor: dict, variant: str): + record = db.get_attachment_record(parse_uuid(attachment_id, "attachment_id"), actor["id"]) + if variant != "original": + raise ApiError(404, "not_found", "Вариант не найден") + data = storage.get_bytes(record["object_key"]) + if len(data) != record["size_bytes"] or hashlib.sha256(data).hexdigest() != record["sha256"]: + raise ApiError(503, "unavailable", "Объект хранилища не совпал с метаданными") + return record["content_type"], data + + +def list_messages(chat_id: str, actor: dict, limit: int, cursor) -> dict: + cid = parse_uuid(chat_id, "chat_id") + decoded = decode_cursor(cursor, cid) if cursor else None + page = db.list_messages(cid, actor["id"], limit, decoded) + next_cursor = None + if page["next_cursor_parts"] is not None: + created, message_id = page["next_cursor_parts"] + next_cursor = encode_cursor(cid, db.format_ts(created), message_id) + return {"items": page["items"], "next_cursor": next_cursor} + + +class FastThreadingHTTPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): + daemon_threads = True + block_on_close = False + allow_reuse_address = True + request_queue_size = 256 + + +class Handler(BaseHTTPRequestHandler): + server_version = "mtusi-messenger/4.0" + protocol_version = "HTTP/1.0" + + def log_message(self, fmt, *args): + return + + def _request_id(self) -> str: + return self.headers.get("X-Request-Id") or str(uuid.uuid4()) + + def _send_json(self, status: int, payload, request_id: str, extra_headers=None): + body = b"" + if status != 204: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + self.send_response(status) + if status != 204: + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.send_header("Connection", "close") + for key, value in (extra_headers or {}).items(): + self.send_header(key, value) + self.end_headers() + if body: + self.wfile.write(body) + + def _send_bytes(self, content_type: str, payload: bytes, request_id: str): + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.send_header("Connection", "close") + self.end_headers() + if payload: + self.wfile.write(payload) + + def _send_error(self, status: int, code: str, message: str, request_id: str, extra_headers=None): + self._send_json( + status, + {"error": {"code": code, "message": message, "request_id": request_id}}, + request_id, + extra_headers, + ) + + def _read_body(self): + length = self.headers.get("Content-Length") + if not length: + return None + try: + size = int(length) + except ValueError as exc: + raise ApiError(400, "invalid_request", "Некорректный Content-Length") from exc + if size > MAX_BODY: + raise ApiError(413, "payload_too_large", "Тело слишком большое") + raw = self.rfile.read(size) + ctype = (self.headers.get("Content-Type") or "").split(";")[0].strip() + if ctype and ctype != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + if not raw: + raise ApiError(400, "invalid_request", "Пустое тело") + try: + return json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ApiError(400, "invalid_request", "Malformed JSON") from exc + + def _read_upload(self) -> tuple[str, bytes]: + length = self.headers.get("Content-Length") + if length is None: + raise ApiError(400, "invalid_request", "Нужен Content-Length") + try: + size = int(length) + except ValueError as exc: + raise ApiError(400, "invalid_request", "Некорректный Content-Length") from exc + if size > max_upload_bytes(): + raise ApiError(413, "payload_too_large", "Файл больше допустимого размера") + if size < 1: + raise ApiError(415, "unsupported_media_type", "Пустое тело") + declared = (self.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower() + if declared not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "Ожидается image/png или image/jpeg") + data = self.rfile.read(size) + if len(data) != size: + raise ApiError(400, "invalid_request", "Тело короче Content-Length") + return declared, data + + def do_GET(self): + self._dispatch("GET") + + def do_POST(self): + self._dispatch("POST") + + def do_DELETE(self): + self._dispatch("DELETE") + + def _dispatch(self, method: str): + global _inflight + with _inflight_lock: + _inflight += 1 + rid = self._request_id() + parsed = urlparse(self.path) + path = parsed.path.rstrip("/") or "/" + query = parse_qs(parsed.query, keep_blank_values=True) + try: + self._route(method, path, query, rid) + except ApiError as exc: + headers = {} + if exc.status == 401: + headers["WWW-Authenticate"] = challenge_for(path) + self._send_error(exc.status, exc.code, exc.message, rid, headers) + except BrokenPipeError: + pass + except Exception: + sys.stderr.write("internal error on %s %s\n" % (method, path)) + self._send_error(500, "internal_error", "Внутренняя ошибка", rid) + finally: + with _inflight_lock: + _inflight -= 1 + + def _route(self, method: str, path: str, query: dict, rid: str): + if method == "GET" and path == "/health/live": + return self._send_json(200, {"status": "ok"}, rid) + if method == "GET" and path == "/health/ready": + if _draining.is_set(): + raise ApiError(503, "unavailable", "Реплика останавливается") + if db.ping() and sessions.ping() and storage.ping(): + return self._send_json(200, {"status": "ok"}, rid) + raise ApiError(503, "unavailable", "Зависимость недоступна") + if _draining.is_set(): + raise ApiError(503, "unavailable", "Реплика останавливается") + + if method == "POST" and path == "/api/v1/users": + return self._send_json(201, create_user(self._read_body()), rid) + if method == "POST" and path == "/api/v1/auth/sessions": + return self._send_json(201, create_session(self.headers), rid) + if method == "DELETE" and path == "/api/v1/auth/sessions/current": + logout(self.headers) + return self._send_json(204, None, rid) + if method == "GET" and path == "/api/v1/users/me": + return self._send_json(200, current_user(self.headers), rid) + + if method == "POST" and path == "/api/v1/chats": + actor = current_user(self.headers) + return self._send_json(201, create_chat(self._read_body(), actor), rid) + if method == "GET" and path == "/api/v1/chats": + actor = current_user(self.headers) + limit = parse_limit(query.get("limit", [None])[0]) + cursor = parse_optional_cursor(query.get("cursor", [None])[0]) + return self._send_json(200, list_chats(actor, limit, cursor), rid) + + match = re.fullmatch(r"/api/v1/chats/([^/]+)", path) + if match and method == "GET": + actor = current_user(self.headers) + return self._send_json(200, db.get_chat(parse_uuid(match.group(1), "chat_id"), actor["id"]), rid) + + match = re.fullmatch(r"/api/v1/chats/([^/]+)/attachments", path) + if match and method == "POST": + actor = current_user(self.headers) + chat_id = parse_uuid(match.group(1), "chat_id") + if not db.is_member(chat_id, actor["id"]): + raise ApiError(404, "not_found", "Чат не найден") + names = query.get("filename") or [] + if len(names) != 1: + raise ApiError(400, "invalid_request", "Нужен один filename") + filename = parse_filename(names[0]) + declared, data = self._read_upload() + body = upload_attachment(chat_id, actor, filename, declared, data) + return self._send_json(201, body, rid) + + match = re.fullmatch(r"/api/v1/attachments/([^/]+)/content", path) + if match and method == "GET": + actor = current_user(self.headers) + variants = query.get("variant") or ["original"] + if len(variants) != 1 or not isinstance(variants[0], str) or variants[0] == "": + raise ApiError(400, "invalid_request", "Некорректный variant") + content_type, payload = read_attachment_bytes(match.group(1), actor, variants[0]) + return self._send_bytes(content_type, payload, rid) + + match = re.fullmatch(r"/api/v1/attachments/([^/]+)", path) + if match and method == "GET": + actor = current_user(self.headers) + return self._send_json(200, read_attachment(match.group(1), actor), rid) + + match = re.fullmatch(r"/api/v1/chats/([^/]+)/messages", path) + if match and method == "POST": + actor = current_user(self.headers) + idem_key = parse_idempotency_key(self.headers) + body = create_message(match.group(1), self._read_body(), actor, idem_key) + return self._send_json(201, body, rid) + if match and method == "GET": + actor = current_user(self.headers) + limit = parse_limit(query.get("limit", [None])[0]) + cursor = parse_optional_cursor(query.get("cursor", [None])[0]) + return self._send_json(200, list_messages(match.group(1), actor, limit, cursor), rid) + + raise ApiError(404, "not_found", "Маршрут не найден") + + +def _inflight_now() -> int: + with _inflight_lock: + return _inflight + + +def main(): + try: + db.init() + except Exception: + sys.stderr.write("database is not configured\n") + try: + sessions.init() + except Exception: + sys.stderr.write("session store is not configured\n") + try: + storage.init() + except Exception: + sys.stderr.write("object storage is not configured\n") + + server = FastThreadingHTTPServer((APP_HOST, APP_PORT), Handler) + + def drain_and_stop(): + time.sleep(drain_seconds()) + deadline = time.monotonic() + 2 + while time.monotonic() < deadline and _inflight_now() > 0: + time.sleep(0.05) + server.shutdown() + + def shutdown(signum, frame): + if _draining.is_set(): + return + _draining.set() + sys.stderr.write("SIGTERM instance=%s stop accepting\n" % INSTANCE_ID) + threading.Thread(target=drain_and_stop, daemon=True).start() + + signal.signal(signal.SIGTERM, shutdown) + signal.signal(signal.SIGINT, shutdown) + sys.stderr.write( + "listening on %s:%s env=%s instance=%s\n" % (APP_HOST, APP_PORT, APP_ENV, INSTANCE_ID) + ) + try: + server.serve_forever(poll_interval=0.1) + finally: + server.server_close() + db.close() + sys.stderr.write("stopped instance=%s\n" % INSTANCE_ID) + + +if __name__ == "__main__": + main() diff --git a/src/sessions.py b/src/sessions.py new file mode 100644 index 00000000..6e7bc918 --- /dev/null +++ b/src/sessions.py @@ -0,0 +1,85 @@ +"""Сессии в Valkey. Ключ — SHA-256 токена, TTL равен остатку жизни сессии. + +Источник истины сессий — общий Valkey, не память процесса API. +AOF и volume сохраняют ключ при перезапуске хранилища. +""" +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + +import redis + +from errors import ApiError + +_client: redis.Redis | None = None + + +def session_ttl() -> int: + raw = os.environ.get("SESSION_TTL_SECONDS", "3600") + try: + value = int(raw) + except ValueError: + return 3600 + return value if value >= 1 else 1 + + +def redis_url() -> str: + path = os.environ.get("CACHE_URL_FILE", "").strip() + if path and Path(path).is_file(): + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get("REDIS_URL", "").strip() + + +def init() -> None: + global _client + url = redis_url() + if not url: + _client = None + return + _client = redis.Redis.from_url( + url, socket_connect_timeout=0.5, socket_timeout=0.5, + decode_responses=True) + + +def _key(token: str) -> str: + digest = hashlib.sha256(token.encode("utf-8")).hexdigest() + return "session:" + digest + + +def _require() -> redis.Redis: + if _client is None: + raise ApiError(503, "unavailable", "Хранилище сессий недоступно") + return _client + + +def ping() -> bool: + try: + return bool(_require().ping()) + except (ApiError, redis.RedisError): + return False + + +def create(token: str, user_id: str, ttl: int) -> None: + try: + _require().set(_key(token), user_id, ex=ttl) + except (ApiError, redis.RedisError) as exc: + raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc + + +def user_id(token: str) -> str: + try: + value = _require().get(_key(token)) + except (ApiError, redis.RedisError) as exc: + raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc + if not value: + raise ApiError(401, "unauthorized", "Сессия недействительна") + return value + + +def revoke(token: str) -> None: + try: + _require().delete(_key(token)) + except (ApiError, redis.RedisError) as exc: + raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..65759e33 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,110 @@ +"""Приватный S3-совместимый bucket. Ключ объекта задаёт сервер, не клиент.""" +from __future__ import annotations + +import io +import os +import sys +from pathlib import Path + +from minio import Minio + +from errors import ApiError + +_client: Minio | None = None + + +def _read_setting(env_name: str, file_env: str) -> str: + path = os.environ.get(file_env, "").strip() + if path and Path(path).is_file(): + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get(env_name, "").strip() + + +def endpoint() -> str: + return os.environ.get("S3_ENDPOINT", "http://s3:9000").strip() + + +def bucket() -> str: + return os.environ.get("S3_BUCKET", "messenger-attachments").strip() + + +def _host_and_secure() -> tuple[str, bool]: + raw = endpoint() + secure = raw.startswith("https://") + host = raw.split("://", 1)[-1].rstrip("/") + return host, secure + + +def init() -> None: + global _client + host, secure = _host_and_secure() + access = _read_setting("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE") + secret = _read_setting("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE") + if not host or not access or not secret or not bucket(): + raise RuntimeError("object storage is not configured") + _client = Minio( + host, + access_key=access, + secret_key=secret, + secure=secure, + region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1", + ) + + +def _api() -> Minio: + if _client is None: + raise ApiError(503, "unavailable", "Хранилище недоступно") + return _client + + +def ping() -> bool: + try: + return _api().bucket_exists(bucket()) + except Exception: + return False + + +def ensure_bucket() -> None: + api = _api() + name = bucket() + if not api.bucket_exists(name): + api.make_bucket(name) + + +def put_bytes(key: str, data: bytes, content_type: str) -> None: + try: + _api().put_object( + bucket(), + key, + io.BytesIO(data), + length=len(data), + content_type=content_type, + ) + except ApiError: + raise + except Exception as exc: + sys.stderr.write("s3 put failed\n") + raise ApiError(503, "unavailable", "Хранилище недоступно") from exc + + +def get_bytes(key: str) -> bytes: + response = None + try: + response = _api().get_object(bucket(), key) + return response.read() + except ApiError: + raise + except Exception as exc: + sys.stderr.write("s3 get failed\n") + raise ApiError(503, "unavailable", "Хранилище недоступно") from exc + finally: + if response is not None: + response.close() + response.release_conn() + + +def delete_quiet(key: str) -> None: + try: + _api().remove_object(bucket(), key) + except Exception: + sys.stderr.write("s3 delete failed\n")