diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..763520f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.11.9-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +WORKDIR /app +RUN groupadd --system app && useradd --system --gid app --home-dir /app app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY app ./app +COPY migrations ./migrations +COPY scripts ./scripts +COPY tests ./tests +COPY contracts ./contracts +COPY lab.json . +COPY alembic.ini . +RUN chown -R app:app /app +USER app +EXPOSE 8443 +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8443", "--ssl-keyfile", "/run/tls/api.key", "--ssl-certfile", "/run/tls/api.crt", "--ssl-ca-certs", "/run/tls/lb-ca.crt", "--ssl-cert-reqs", "2", "--timeout-keep-alive", "5", "--timeout-graceful-shutdown", "20"] diff --git a/REPORT.md b/REPORT.md index 9801db4..944d212 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,150 @@ # Отчёт — лабораторная 7 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +## Сведения о работе -## Запуск из чистого clone +- Автор, группа: Перевалова Виктория, БПИ2302. +- Целевая оценка: 3. +- Выбранный трек 5: не заявлен. +- Commit/PR: не указан. +- Использованные технологии: Python 3.11, FastAPI, SQLAlchemy, PostgreSQL, Redis, RabbitMQ, SeaweedFS/S3 API, OpenBao 2.2.0, Docker Compose, Nginx, TLS/stunnel, pytest/скрипты проверки проекта. +- Помощь ИИ: использовалась при анализе требований, диагностике, проверке конфигурации и подготовке документации. Реализация и результаты проверок должны быть подтверждены локальным стендом. -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +## Запуск и эксплуатация + +Рабочее окружение — Windows PowerShell, Docker Compose. Проверки выполнялись на локальном стенде с внешним HTTPS endpoint `https://localhost:8443` и CA-файлом `secrets/tls/ca.crt`. + +### Предварительные условия + +- Docker Desktop с поддержкой Docker Compose. +- Python 3.11 для запуска локальных проверочных скриптов. +- Локально подготовленные `.env` и TLS secrets/certificates согласно `README.md`, `infra/README.md`, `infra/openbao/README.md` и `contracts/README.md`. +- TLS private keys, CA private key, OpenBao root token, unseal shares и AppRole SecretID должны создаваться/передаваться оператором вне Git. Секреты в отчёт не включаются. + +### Запуск существующего стенда + +Из корня репозитория: + +```powershell +docker compose up -d --build +docker compose ps +``` + +Проверка внешней готовности: + +```powershell +Invoke-WebRequest ` + -Uri https://localhost:8443/health/ready ` + -Certificate (Get-Item .\secrets\tls\ca.crt) +``` + +Для Python-проверок: + +```powershell +python .\tests\smoke.py --base-url https://localhost:8443 --ca-file .\secrets\tls\ca.crt +python .\tests\security.py --base-url https://localhost:8443 --ca-file .\secrets\tls\ca.crt +python .\tests\test_login_rate_limit.py --base-url https://localhost:8443 --ca-file .\secrets\tls\ca.crt +``` + +Команда `security.py` проверяет внешний HTTPS и отказ при пустом trust store. Внутренний mTLS и OpenBao ACL проверялись отдельными сценариями, описанными ниже. + +### Остановка + +Обычная остановка без удаления постоянных данных: + +```powershell +docker compose stop +``` + +Для последующего запуска: + +```powershell +docker compose start +``` + +Удаление volumes намеренно не выполнялось. Команду `docker compose down -v` нельзя использовать как обычную остановку: она удаляет постоянные данные стенда. + +> Для воспроизведения из чистого clone необходимо выполнить локальный bootstrap секретов и конфигурации по README-файлам проекта. Точные значения секретов в отчёте не приводятся. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Стенд развёрнут контейнерами Docker Compose. Внешний REST-трафик проходит через Nginx ingress по HTTPS на `127.0.0.1:8443`. API не публикует собственный бизнес-порт на хост. Внутренний доступ LB → API защищён mTLS: API проверяет клиентский сертификат и разрешённую URI SAN identity `spiffe://lab7/ingress/nginx`. + +Основные компоненты: +- Nginx — внешний TLS ingress и проксирование запросов к API. +- API — FastAPI-приложение; две реплики в Compose. +- PostgreSQL — постоянное хранилище пользователей, сессий, чатов, сообщений, вложений, задач и outbox-записей. +- Redis — вспомогательное хранилище/координация, в том числе для ограничения частоты входа. +- RabbitMQ — брокер задач обработки вложений. +- SeaweedFS/S3 API — хранение исходных файлов и производных объектов. +- Worker — асинхронная обработка изображений. +- Outbox publisher — публикация событий из transactional outbox. +- OpenBao — хранение секретов с сервисными AppRole identities и отдельными policy. +- TLS proxy/stunnel — TLS-граница для внутренних соединений, где приложение/сервис использует локальный proxy hop. +- Telemetry/Collector — сбор телеметрии согласно конфигурации infra. + +Вложения загружаются в staging, для них создаются записи БД и задания обработки. Worker забирает задания из брокера, обрабатывает изображения и сохраняет варианты в S3-совместимое хранилище. Метаданные и состояние обработки хранятся в PostgreSQL. Доступ к вложению проверяется по членству пользователя в чате. Для загрузки действует предел 10 MiB: проверяется и заявленный `Content-Length`, и фактически прочитанный размер потока; при превышении возвращается HTTP 413. + +Пароли пользователей хешируются Argon2id. Проверенная конфигурация: `time_cost=2`, `memory_cost=19456 KiB`, `parallelism=1`. Ограничение входа настроено на 5 неудачных попыток в окне 60 секунд; при превышении API возвращает HTTP 429 и `Retry-After`. + +OpenBao работает не в dev mode, использует persistent storage и TLS listener. API, Worker и Outbox используют отдельные AppRole credentials и scoped policies. Проверены чтение собственного пути (HTTP 200), отказ чтения чужого пути (HTTP 403) и отказ записи (HTTP 403). Приложения подключаются к OpenBao по HTTPS с проверкой CA. + +### Контейнерная изоляция + +По результатам `docker inspect`: +- API: пользователь `app`, `read_only=true`, `privileged=false`, `CapDrop=ALL`, лимит памяти 768 MiB, CPU 1. +- Worker: пользователь `app`, `read_only=true`, `privileged=false`, `CapDrop=ALL`, лимит памяти 1 GiB, CPU 1. +- Outbox publisher: пользователь `app`, `read_only=true`, `privileged=false`, `CapDrop=ALL`, лимит памяти 384 MiB, CPU 0.5. +- Nginx: пользователь `101:101`, `privileged=false`, `CapDrop=ALL`. +- Все проверенные контейнеры не используют host network. В `compose.yaml` и `Dockerfile` не обнаружены ссылки на Docker socket, `network_mode: host` или `privileged: true`. +- API port не опубликован напрямую на хост; внешняя точка входа — Nginx на loopback `127.0.0.1:8443`. Административные endpoints OpenBao и вспомогательные порты привязаны к loopback. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда / сценарий | Наблюдаемый результат | Статус | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Argon2id | Проверка `app/services/user_service.py` и runtime env API | Argon2id; time_cost=2, memory_cost=19456 KiB, parallelism=1 | PASS | +| Внешний HTTPS | `python .\tests\security.py --base-url https://localhost:8443 --ca-file .\secrets\tls\ca.crt` | Доверенный CA/hostname принимается; пустой trust store отклонён именно ошибкой проверки сертификата | PASS | +| Публичный API smoke | `python .\tests\smoke.py --base-url https://localhost:8443 --ca-file .\secrets\tls\ca.crt` | `PASS: lab7 public API scenario`; проверены пользователи, чаты, сообщения, ошибки доступа, вложения и logout | PASS | +| Login rate limit | `python .\tests\test_login_rate_limit.py --base-url https://localhost:8443 --ca-file .\secrets\tls\ca.crt` | `PASS: login rate limit returns 429 and Retry-After` | PASS | +| Размер вложения | Анализ `app/api/attachments.py`, `read_limited()` и вызова в `upload_attachment()` | Проверяются Content-Length и реальный поток; превышение MAX_BYTES (10 MiB) приводит к HTTP 413. Отдельный runtime negative upload test не зафиксирован | Реализация подтверждена; отдельный тест не заявлен | +| Доступ к вложениям | Smoke: GET metadata/content от участника и постороннего пользователя | Для постороннего пользователя metadata и content возвращают 404 | PASS | +| Logout | Smoke: DELETE `/api/v1/auth/sessions/current`, затем GET `/api/v1/users/me` | Logout возвращает 204; повторный запрос со старой сессией возвращает 401 | PASS | +| Подмена пользователя | Smoke: запрос с `X-User-Id` другого пользователя | Bearer identity не меняется | PASS | +| Внутренний mTLS | Временный probe в API container: no-cert, foreign-CA, wrong-identity, positive | Без сертификата и с чужим CA — TLS отказ; Worker identity от доверенного CA — отказ API; ingress identity — успех | PASS | +| OpenBao TLS | Запрос из API container к `https://openbao:8200/v1/sys/health` с `verify=/run/tls/ca.crt` | HTTP 200, TLS verification PASS | PASS | +| OpenBao ACL | Проверки AppRole API/Worker/Outbox: свой path, чужой path, запись | Свой path HTTP 200; чужой path и запись HTTP 403 | PASS | +| Persistence | `python .\tests\resilience_windows.py --base-url https://localhost:8443 --ca-file .\secrets\tls\ca.crt --mode persistence --action-script .\scripts\restart.sh` | Подтверждённые данные и существующая сессия сохранены после перезапуска; тест сообщил, что scope hook требует операторского подтверждения | PASS | +| API failover | `python .\tests\resilience_windows.py --base-url https://localhost:8443 --ca-file .\secrets\tls\ca.crt --mode failover --action-script .\scripts\stop-one.sh` | 2 наблюдаемые реплики; 5 успешных записей; 1 переходная ошибка; восстановление за 6.297 s; подтверждённые данные и сессия сохранены | PASS | +| Docker hardening | `docker inspect` контейнеров и поиск по compose/Dockerfile | API/Worker/Outbox non-root, read-only, CapDrop ALL, CPU/RAM limits; privileged=false; host network и Docker socket не обнаружены | PASS | +| Отсутствие прямого обхода ingress | `docker compose ps` / `docker inspect` port bindings | API не опубликован на host; Nginx слушает `127.0.0.1:8443` | PASS | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +### Ограничения доказательств + +- Для persistence/failover проверены результаты сценариев, но тестовая утилита отдельно предупреждает, что реальный scope инфраструктурного hook требует операторского подтверждения. Поэтому не утверждается, что каждый backend-компонент был перезапущен тестом. +- Временный mTLS probe выполнялся в контейнере API и не изменял исходный тестовый файл проекта. Исходный `scripts/test-mtls.ps1` не считается полностью пройденным; в частности, проверка запрещённой identity выполнялась отдельным корректно направленным запросом непосредственно к API. +- Отдельный отрицательный runtime-тест на upload больше 10 MiB в зафиксированных результатах отсутствует; подтверждены реализация и точка вызова ограничения. +- Успешные проверки локального учебного стенда не являются аудитом production-безопасности. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Ожидаемое поведение | Наблюдение | Восстановление / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| HTTPS без доверенного CA | TLS handshake должен завершиться ошибкой верификации | Соединение с правильным CA прошло; пустой trust store отклонён ошибкой сертификата | Отказ корректный | +| mTLS без клиентского сертификата | API не должен принимать соединение | TLS handshake отклонён | Отказ корректный | +| mTLS с сертификатом чужого CA | API не должен принимать соединение | TLS handshake отклонён | Отказ корректный | +| mTLS с доверенным, но запрещённым URI SAN `spiffe://lab7/worker` | API должен отклонить identity | API отклонил запрос | Отказ корректный | +| mTLS с разрешённой ingress identity | API должен принять запрос | Запрос принят | Успех | +| Превышение login rate limit | HTTP 429 + Retry-After | Отдельный тест завершился PASS | Доступ ограничивается на время окна | +| Остановка одной API-реплики | Вторая реплика продолжает обслуживать запросы | 5 успешных записей, 1 переходная ошибка, восстановление за 6.297 s | Подтверждённые данные и сессия сохранены | +| Перезапуск по persistence hook | Сохранение подтверждённых данных и сессии | Данные и сессия сохранены | Потери подтверждённых данных не обнаружено; scope hook требует операторского подтверждения | + +Нагрузочные RPS и p50/p95/p99 в рамках приведённых доказательств не измерялись; численные показатели производительности не заявляются. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +1. Зафиксировать операторские команды и состояние каждого контейнера/хранилища до и после persistence hook. +2. Сохранить обезличенные логи/выводы проверок в `evidence/`, не включая токены, пароли, SecretID, приватные ключи и unseal shares. +3. При необходимости расширить проверку TLS/ACL для каждого внутреннего backend и документировать точные server SAN, service identity, ACL и сроки credentials по фактической конфигурации. +4. Провести отдельный нагрузочный эксперимент с описанием профиля, concurrency, длительности и ограничений генератора. + +Для заявленной целевой оценки 3 приведены подтверждённые проверки HTTPS, mTLS, OpenBao scoped policies, Argon2id, rate limit, негативного доступа, контейнерного hardening и отказоустойчивости API. Непроверенные расширения не заявляются как выполненные. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..421e3fe --- /dev/null +++ b/alembic.ini @@ -0,0 +1,117 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +# Use forward slashes (/) also on windows to provide an os agnostic path +script_location = migrations + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +# version_path_separator = newline +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/api/attachments.py b/app/api/attachments.py new file mode 100644 index 0000000..44b01b8 --- /dev/null +++ b/app/api/attachments.py @@ -0,0 +1,645 @@ +import hashlib +import tempfile +import uuid +from datetime import datetime, timezone, timedelta +from urllib.parse import quote + +from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, Response +from fastapi.responses import StreamingResponse +from sqlalchemy import select, func, text +from sqlalchemy.orm import Session +from botocore.exceptions import BotoCoreError, ClientError +from opentelemetry.propagate import inject + +from app.api.deps import get_db, get_current_user +from app.core.config import settings +from app.models import CreateAttachmentUpload +from app.db.models import ( + Attachment, + AttachmentUpload, + AttachmentVariant, + ProcessingJob, + OutboxEvent, + User, +) +from app.services.object_storage import client, create_presigned_upload_url +from app.services.image_validation import MAX_BYTES, validate_image_file +from app.repositories.chat_repository import ChatRepository + +router = APIRouter(prefix="/api/v1", tags=["Attachments"]) + + +def iso_utc(value): + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def attachment_json(item, db): + variants = db.scalars( + select(AttachmentVariant) + .where(AttachmentVariant.attachment_id == item.id) + .order_by(AttachmentVariant.variant) + ).all() + + return { + "id": str(item.id), + "chat_id": str(item.chat_id), + "owner_id": str(item.owner_id), + "filename": item.filename, + "content_type": item.content_type, + "size_bytes": item.size_bytes, + "sha256": item.sha256, + "status": item.status, + "variants": [ + { + "name": variant.variant, + "content_type": variant.content_type, + "size_bytes": variant.size_bytes, + "width": variant.width, + "height": variant.height, + "sha256": variant.sha256, + } + for variant in variants + ], + "error_code": item.error_code, + "created_at": iso_utc(item.created_at), + "updated_at": iso_utc(item.updated_at), + } + + +def member_attachment(db, attachment_id, user_id): + item = db.get(Attachment, attachment_id) + if item is None or not ChatRepository(db).is_member(item.chat_id, user_id): + raise HTTPException(404, "Attachment not found") + return item + + +async def read_limited(request: Request): + declared = request.headers.get("content-length") + if declared and declared.isdigit() and int(declared) > MAX_BYTES: + raise HTTPException(413, "Image exceeds 10 MiB") + size = 0 + # Keep at most 1 MiB in RAM; larger bodies spill to a temporary file. + spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b") + try: + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise HTTPException(413, "Image exceeds 10 MiB") + spool.write(chunk) + spool.seek(0) + return spool, size + except Exception: + spool.close() + raise + + +def sha256_file(fileobj): + digest = hashlib.sha256() + fileobj.seek(0) + while True: + chunk = fileobj.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + fileobj.seek(0) + return digest.hexdigest() + + +@router.post("/chats/{chat_id}/attachments/presigned", status_code=201) +def create_attachment_upload( + chat_id: str, + payload: CreateAttachmentUpload, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + try: + chat_uuid = uuid.UUID(chat_id) + except ValueError: + raise HTTPException(400, "Invalid chat UUID") from None + + if not ChatRepository(db).is_member(chat_uuid, user.id): + raise HTTPException(404, "Chat not found") + + if payload.size_bytes > settings.max_upload_bytes: + raise HTTPException(413, "Image exceeds configured upload limit") + + upload_id = uuid.uuid4() + object_key = f"uploads/{chat_uuid}/{upload_id}" + expires_at = datetime.now(timezone.utc) + timedelta( + seconds=settings.presigned_upload_ttl_seconds + ) + + try: + upload_url = create_presigned_upload_url( + object_key=object_key, + content_type=payload.content_type, + ) + except (BotoCoreError, ClientError): + raise HTTPException(503, "Object storage unavailable") from None + + upload = AttachmentUpload( + id=upload_id, + chat_id=chat_uuid, + owner_id=user.id, + filename=payload.filename, + content_type=payload.content_type, + expected_size_bytes=payload.size_bytes, + expected_sha256=payload.sha256.lower(), + object_key=object_key, + status="pending", + expires_at=expires_at, + ) + + try: + db.add(upload) + db.commit() + except Exception: + db.rollback() + raise HTTPException(503, "Could not save upload metadata") from None + + return { + "upload_id": str(upload_id), + "upload_url": upload_url, + "method": "PUT", + "headers": { + "Content-Type": payload.content_type, + }, + "expires_in": settings.presigned_upload_ttl_seconds, + "expires_at": iso_utc(expires_at), + } + + +@router.post("/attachments/uploads/{upload_id}/complete", status_code=200) +def complete_attachment_upload( + upload_id: str, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + try: + parsed_id = uuid.UUID(upload_id) + except ValueError: + raise HTTPException(400, "Invalid upload UUID") from None + + # Lock the upload row to prevent concurrent completion. + upload = db.scalar( + select(AttachmentUpload) + .where(AttachmentUpload.id == parsed_id) + .with_for_update() + ) + + if upload is None: + raise HTTPException(404, "Upload not found") + + if ( + upload.owner_id != user.id + or not ChatRepository(db).is_member(upload.chat_id, user.id) + ): + raise HTTPException(404, "Upload not found") + + # Idempotent completion: return the existing attachment. + if upload.status == "completed": + item = db.scalar( + select(Attachment).where( + Attachment.object_key == upload.object_key + ) + ) + if item is None: + raise HTTPException(409, "Completed attachment not found") + return attachment_json(item, db) + + if upload.status != "pending": + raise HTTPException(409, "Upload is not pending") + + now = datetime.now(timezone.utc) + + if upload.expires_at <= now: + upload.status = "expired" + db.commit() + + try: + client().delete_object( + Bucket=settings.s3_bucket, + Key=upload.object_key, + ) + except Exception: + pass + + raise HTTPException(409, "Upload has expired") + + s3 = client() + + # Read the object from S3. + try: + result = s3.get_object( + Bucket=settings.s3_bucket, + Key=upload.object_key, + ) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + + if code in {"NoSuchKey", "404", "NotFound"}: + raise HTTPException(409, "Uploaded object not found") from None + + raise HTTPException(503, "Object storage unavailable") from None + except BotoCoreError: + raise HTTPException(503, "Object storage unavailable") from None + + body = result["Body"] + spool = tempfile.SpooledTemporaryFile( + max_size=1024 * 1024, + mode="w+b", + ) + + def reject_upload(status_code: int, detail: str): + upload.status = "failed" + db.commit() + + try: + s3.delete_object( + Bucket=settings.s3_bucket, + Key=upload.object_key, + ) + except Exception: + pass + + raise HTTPException(status_code=status_code, detail=detail) + + try: + actual_size = result.get("ContentLength") + + if actual_size != upload.expected_size_bytes: + reject_upload(409, "Uploaded file size mismatch") + + if actual_size > settings.max_upload_bytes: + reject_upload(413, "Image exceeds configured upload limit") + + total = 0 + digest = hashlib.sha256() + + try: + while True: + chunk = body.read(64 * 1024) + if not chunk: + break + + total += len(chunk) + + if total > settings.max_upload_bytes: + reject_upload(413, "Image exceeds configured upload limit") + + if total > upload.expected_size_bytes: + reject_upload(409, "Uploaded file size mismatch") + + digest.update(chunk) + spool.write(chunk) + finally: + body.close() + + if total != upload.expected_size_bytes: + reject_upload(409, "Uploaded file size mismatch") + + if digest.hexdigest() != upload.expected_sha256: + reject_upload(409, "Uploaded file checksum mismatch") + + # Validate the actual image content, not only S3 metadata. + try: + actual_type, _, _ = validate_image_file( + spool, + upload.content_type, + ) + except HTTPException as exc: + reject_upload(exc.status_code, str(exc.detail)) + + attachment = Attachment( + id=uuid.uuid4(), + chat_id=upload.chat_id, + owner_id=upload.owner_id, + filename=upload.filename, + content_type=actual_type, + size_bytes=total, + sha256=digest.hexdigest(), + object_key=upload.object_key, + status="ready", + error_code=None, + ) + + upload.status = "completed" + upload.completed_at = now + + db.add(attachment) + db.commit() + db.refresh(attachment) + + return attachment_json(attachment, db) + + except HTTPException: + raise + except (BotoCoreError, ClientError): + db.rollback() + raise HTTPException(503, "Object storage unavailable") from None + except Exception: + db.rollback() + raise + finally: + spool.close() + + +@router.post("/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment( + chat_id: str, + request: Request, + filename: str = Query(..., min_length=1, max_length=255), + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + try: + chat_uuid = uuid.UUID(chat_id) + except ValueError: + raise HTTPException(400, "Invalid chat UUID") from None + + if not ChatRepository(db).is_member(chat_uuid, user.id): + raise HTTPException(404, "Chat not found") + + if ( + not filename.strip() + or any(c in filename for c in ("/", "\\", "\x00")) + or any(ord(c) < 32 or ord(c) == 127 for c in filename) + ): + raise HTTPException(400, "Unsafe filename") + + content_type = ( + request.headers.get("content-type", "") + .split(";", 1)[0] + .strip() + .lower() + ) + + if content_type not in {"image/png", "image/jpeg"}: + raise HTTPException(415, "Only PNG and JPEG are supported") + + spool, size = await read_limited(request) + + attachment_id = uuid.uuid4() + job_id = uuid.uuid4() + + staging_key = f"staging/{chat_uuid}/{attachment_id}" + object_key = f"original/{chat_uuid}/{attachment_id}" + + staged = False + + try: + actual_type, _, _ = validate_image_file(spool, content_type) + digest = sha256_file(spool) + + # Serialize admission checks across all API instances. + db.execute( + text("SELECT pg_advisory_xact_lock(:lock_id)"), + {"lock_id": 72420261110}, + ) + + active_jobs = db.scalar( + select(func.count()) + .select_from(ProcessingJob) + .where(ProcessingJob.status.in_(("queued", "processing"))) + ) or 0 + + if active_jobs >= settings.max_active_processing_jobs: + raise HTTPException( + status_code=429, + detail="Processing queue is full; retry later", + headers={"Retry-After": "5"}, + ) + + s3 = client() + + try: + spool.seek(0) + s3.put_object( + Bucket=settings.s3_bucket, + Key=staging_key, + Body=spool, + ContentType=actual_type, + ) + staged = True + except (BotoCoreError, ClientError): + raise HTTPException( + 503, "Object storage unavailable" + ) from None + + now = datetime.now(timezone.utc) + + attachment = Attachment( + id=attachment_id, + chat_id=chat_uuid, + owner_id=user.id, + filename=filename, + content_type=actual_type, + size_bytes=size, + sha256=digest, + object_key=object_key, + staging_key=staging_key, + status="queued", + error_code=None, + pipeline_version=1, + ) + + job = ProcessingJob( + id=job_id, + attachment_id=attachment_id, + pipeline_version=1, + attempt=0, + status="queued", + available_at=now, + ) + + # Preserve the current HTTP trace context in the durable outbox event. + trace_context = {} + inject(trace_context) + + event = OutboxEvent( + event_type="attachment.process.v1", + aggregate_id=attachment_id, + payload={ + "schema_version": 1, + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": 1, + "attempt": 1, + "created_at": now.isoformat(), + "trace_context": trace_context, + }, + status="pending", + ) + + db.add_all([attachment, job, event]) + db.commit() + db.refresh(attachment) + + return attachment_json(attachment, db) + + except HTTPException: + db.rollback() + if staged: + try: + client().delete_object( + Bucket=settings.s3_bucket, + Key=staging_key, + ) + except Exception: + pass + raise + + except Exception: + db.rollback() + + if staged: + try: + client().delete_object( + Bucket=settings.s3_bucket, + Key=staging_key, + ) + except Exception: + pass + + raise HTTPException( + 503, "Could not save attachment processing job" + ) from None + + finally: + spool.close() + + +@router.get("/attachments/{attachment_id}") +def get_attachment( + attachment_id: str, + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + try: + parsed = uuid.UUID(attachment_id) + except ValueError: + raise HTTPException(400, "Invalid attachment UUID") from None + + item = member_attachment(db, parsed, user.id) + return attachment_json(item, db) + + +@router.get("/attachments/{attachment_id}/content") +def download_attachment( + attachment_id: str, + variant: str = "original", + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +): + try: + parsed = uuid.UUID(attachment_id) + except ValueError: + raise HTTPException(400, "Invalid attachment UUID") from None + + item = member_attachment(db, parsed, user.id) + + if item.status != "ready": + raise HTTPException(409, "Attachment is not ready") + + # Select the original file or a generated variant. + if variant == "original": + object_key = item.object_key + expected_size = item.size_bytes + expected_sha256 = item.sha256 + content_type = item.content_type + else: + variant_item = db.scalar( + select(AttachmentVariant).where( + AttachmentVariant.attachment_id == item.id, + AttachmentVariant.variant == variant, + ) + ) + + if variant_item is None: + raise HTTPException(404, "Variant not found") + + object_key = variant_item.object_key + expected_size = variant_item.size_bytes + expected_sha256 = variant_item.sha256 + content_type = variant_item.content_type + + spool = tempfile.SpooledTemporaryFile( + max_size=1024 * 1024, + mode="w+b", + ) + + try: + result = client().get_object( + Bucket=settings.s3_bucket, + Key=object_key, + ) + + body = result["Body"] + total = 0 + digest = hashlib.sha256() + + try: + while True: + chunk = body.read(64 * 1024) + + if not chunk: + break + + total += len(chunk) + + if total > MAX_BYTES: + raise HTTPException( + 503, + "Stored object exceeds configured limit", + ) + + digest.update(chunk) + spool.write(chunk) + finally: + body.close() + + # Verify stored object integrity. + if ( + total != expected_size + or digest.hexdigest() != expected_sha256 + ): + raise HTTPException( + 503, + "Stored object checksum mismatch", + ) + + spool.seek(0) + + except (BotoCoreError, ClientError): + spool.close() + raise HTTPException( + 503, + "Object storage unavailable", + ) from None + + except Exception: + spool.close() + raise + + def stream_verified_file(): + try: + while True: + chunk = spool.read(64 * 1024) + + if not chunk: + break + + yield chunk + finally: + spool.close() + + return StreamingResponse( + stream_verified_file(), + media_type=content_type, + headers={ + "Content-Disposition": ( + f"inline; filename*=UTF-8''{quote(item.filename)}" + ) + }, + ) diff --git a/app/api/auth.py b/app/api/auth.py new file mode 100644 index 0000000..912229d --- /dev/null +++ b/app/api/auth.py @@ -0,0 +1,129 @@ + +from datetime import timezone + +from fastapi import ( + APIRouter, + Depends, + Header, + HTTPException, + Request, + status, +) +from fastapi.security import HTTPBasic, HTTPBasicCredentials +from sqlalchemy.orm import Session + +from app.api.deps import get_db +from app.api.users import user_to_response +from app.services.auth_service import ( + AuthService, + InvalidCredentialsError, + InvalidSessionError, + LoginRateLimitError, + LoginRateLimitUnavailableError, +) + + +router = APIRouter(prefix="/api/v1/auth", tags=["Authentication"]) + +basic_security = HTTPBasic(auto_error=False) + + +@router.post("/sessions", status_code=status.HTTP_201_CREATED) +def create_session( + request: Request, + credentials: HTTPBasicCredentials = Depends(basic_security), + db: Session = Depends(get_db), +): + if credentials is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Basic authentication required", + headers={ + "WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"' + }, + ) + + client_ip = request.headers.get("x-real-ip") + + if not client_ip and request.client: + client_ip = request.client.host + + try: + result = AuthService(db).create_session( + username=credentials.username, + password=credentials.password, + client_ip=client_ip or "unknown", + ) + + expires_at = result["expires_at"] + + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=timezone.utc) + else: + expires_at = expires_at.astimezone(timezone.utc) + + return { + "token": result["token"], + "token_type": result["token_type"], + "expires_at": expires_at.isoformat( + timespec="microseconds" + ).replace("+00:00", "Z"), + "user": user_to_response(result["user"]), + } + + except InvalidCredentialsError: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid username or password", + headers={ + "WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"' + }, + ) from None + + except LoginRateLimitError as exc: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail="Too many login attempts. Try again later.", + headers={"Retry-After": str(exc.retry_after)}, + ) from None + + except LoginRateLimitUnavailableError: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Authentication service temporarily unavailable", + headers={"Retry-After": "5"}, + ) from None + + +@router.delete( + "/sessions/current", + status_code=status.HTTP_204_NO_CONTENT, +) +def delete_current_session( + authorization: str | None = Header(default=None), + db: Session = Depends(get_db), +): + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Bearer token required", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + token = authorization[7:].strip() + + if not token: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Bearer token required", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + try: + AuthService(db).delete_session(token) + except InvalidSessionError: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired session", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) from None diff --git a/app/api/chats.py b/app/api/chats.py new file mode 100644 index 0000000..40c4626 --- /dev/null +++ b/app/api/chats.py @@ -0,0 +1,90 @@ +import uuid + +from fastapi import APIRouter, Depends, HTTPException, Query, status +from sqlalchemy.orm import Session + +from app.api.deps import get_db, get_current_user +from app.db.models import User as UserModel +from app.models import Chat as ChatResponse, CreateChat +from app.services.chat_service import ( + ChatService, + ChatNotFoundError, + InvalidChatDataError, +) + + +router = APIRouter(prefix="/api/v1/chats", tags=["Chats"]) + + +def chat_to_response(chat) -> ChatResponse: + return ChatResponse( + id=str(chat.id), + title=chat.title, + member_ids=[str(member.user_id) for member in chat.members], + created_at=chat.created_at, + ) + + +@router.post("", response_model=ChatResponse, status_code=status.HTTP_201_CREATED) +def create_chat( + body: CreateChat, + db: Session = Depends(get_db), + user: UserModel = Depends(get_current_user), +): + try: + chat = ChatService(db).create_chat( + title=body.title, + member_ids=body.member_ids, + creator_id=user.id, + ) + return chat_to_response(chat) + + except InvalidChatDataError as exc: + raise HTTPException( + status_code=400, + detail=str(exc), + ) from None + + +@router.get("") +def list_chats( + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = None, + db: Session = Depends(get_db), + user: UserModel = Depends(get_current_user), +): + if cursor is not None: + raise HTTPException( + status_code=400, + detail="Chat cursor pagination is not implemented yet", + ) + + chats = ChatService(db).list_chats(user.id) + chats = chats[:limit] + + return { + "items": [chat_to_response(chat) for chat in chats], + "next_cursor": None, + } + + +@router.get("/{chat_id}", response_model=ChatResponse) +def get_chat( + chat_id: str, + db: Session = Depends(get_db), + user: UserModel = Depends(get_current_user), +): + try: + parsed_id = uuid.UUID(chat_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid chat UUID") from None + + try: + chat = ChatService(db).get_chat(parsed_id, user.id) + return chat_to_response(chat) + + except ChatNotFoundError: + raise HTTPException( + status_code=404, + detail="Chat not found", + ) from None diff --git a/app/api/deps.py b/app/api/deps.py new file mode 100644 index 0000000..6c69586 --- /dev/null +++ b/app/api/deps.py @@ -0,0 +1,40 @@ +from fastapi import Depends, HTTPException, status +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from sqlalchemy.orm import Session + +from app.db.session import SessionLocal +from app.services.auth_service import AuthService, InvalidSessionError + + +bearer_scheme = HTTPBearer(auto_error=False) + + +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() + + +def get_current_user( + credentials: HTTPAuthorizationCredentials | None = Depends(bearer_scheme), + db: Session = Depends(get_db), +): + if credentials is None or credentials.scheme.lower() != "bearer": + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Bearer token required", + headers={"WWW-Authenticate": "Bearer"}, + ) + + try: + user = AuthService(db).authenticate(credentials.credentials) + return user + + except InvalidSessionError: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired token", + headers={"WWW-Authenticate": "Bearer"}, + ) from None diff --git a/app/api/messages.py b/app/api/messages.py new file mode 100644 index 0000000..073371b --- /dev/null +++ b/app/api/messages.py @@ -0,0 +1,73 @@ +import re +import uuid + +from fastapi import APIRouter, Depends, Header, HTTPException, Query, status +from sqlalchemy.orm import Session + +from app.api.deps import get_db, get_current_user +from app.db.models import User as UserModel +from app.models import CreateMessage, Message as MessageResponse +from app.services.message_service import ( + MessageService, MessageNotFoundError, InvalidMessageDataError, + IdempotencyConflictError, +) + +router = APIRouter(prefix="/api/v1/chats/{chat_id}/messages", tags=["Messages"]) +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +def message_to_response(message) -> MessageResponse: + return MessageResponse( + id=str(message.id), chat_id=str(message.chat_id), + sender_id=str(message.sender_id), text=message.text, + created_at=message.created_at, attachment_ids=message.attachment_ids or [], + ) + + +def parse_chat_id(chat_id: str) -> uuid.UUID: + try: + return uuid.UUID(chat_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid chat UUID") from None + + +@router.post("", response_model=MessageResponse, status_code=status.HTTP_201_CREATED) +def create_message( + chat_id: str, body: CreateMessage, + idempotency_key: str = Header(..., alias="Idempotency-Key", min_length=1, max_length=128), + db: Session = Depends(get_db), + user: UserModel = Depends(get_current_user), +): + if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise HTTPException(status_code=400, detail="Invalid Idempotency-Key") + parsed_id = parse_chat_id(chat_id) + try: + message = MessageService(db).create_message( + chat_id=parsed_id, sender_id=user.id, text=body.text, + idempotency_key=idempotency_key, attachment_ids=body.attachment_ids, + ) + return message_to_response(message) + except MessageNotFoundError: + raise HTTPException(status_code=404, detail="Chat not found") from None + except InvalidMessageDataError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from None + except IdempotencyConflictError: + raise HTTPException(status_code=409, detail="Idempotency-Key was already used with a different request") from None + + +@router.get("") +def list_messages( + chat_id: str, limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = None, db: Session = Depends(get_db), + user: UserModel = Depends(get_current_user), +): + parsed_id = parse_chat_id(chat_id) + try: + messages, next_cursor = MessageService(db).get_history( + chat_id=parsed_id, user_id=user.id, limit=limit, cursor=cursor + ) + return {"items": [message_to_response(item) for item in messages], "next_cursor": next_cursor} + except MessageNotFoundError: + raise HTTPException(status_code=404, detail="Chat not found") from None + except InvalidMessageDataError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from None diff --git a/app/api/users.py b/app/api/users.py new file mode 100644 index 0000000..67adc7c --- /dev/null +++ b/app/api/users.py @@ -0,0 +1,55 @@ +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy.orm import Session + +from app.api.deps import get_db, get_current_user +from app.db.models import User as UserModel +from app.models import CreateUser, User +from app.services.user_service import ( + UserService, + UsernameAlreadyExistsError, +) + + +router = APIRouter(prefix="/api/v1/users", tags=["Users"]) + + +def user_to_response(user: UserModel) -> User: + return User( + id=str(user.id), + username=user.username, + display_name=user.display_name, + created_at=user.created_at, + ) + + +@router.post("", response_model=User, status_code=status.HTTP_201_CREATED) +def register_user( + body: CreateUser, + db: Session = Depends(get_db), +): + try: + user = UserService(db).register( + username=body.username, + display_name=body.display_name, + password=body.password, + ) + return user_to_response(user) + + except UsernameAlreadyExistsError: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Username already exists", + ) from None + + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from None + + +@router.get("/me", response_model=User) +def get_me( + current_user: UserModel = Depends(get_current_user), +): + return user_to_response(current_user) diff --git a/app/core/__init__.py b/app/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/core/config.py b/app/core/config.py new file mode 100644 index 0000000..ae85c83 --- /dev/null +++ b/app/core/config.py @@ -0,0 +1,78 @@ +import os + +from pydantic import AliasChoices, Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + app_name: str = "Messenger API" + app_env: str = "lab" + database_url: str = "postgresql+psycopg://messenger:messenger@postgres:5432/messenger" + db_pool_size: int = 5 + db_max_overflow: int = 5 + db_pool_timeout: int = 10 + session_ttl_seconds: int = 3600 + cache_url: str = "redis://redis:6379/0" + login_rate_limit_attempts: int = Field(default=5, ge=1, le=20) + login_rate_limit_window_seconds: int = Field(default=60, ge=10, le=3600) + message_cache_ttl_seconds: int = 60 + idempotency_ttl_seconds: int = 86400 + instance_id: str = os.getenv("INSTANCE_ID") or os.getenv("HOSTNAME") or "api-local" + shutdown_grace_seconds: int = 20 + max_active_processing_jobs: int = Field(default=20, ge=1, le=10000) + + s3_endpoint_url: str = Field( + default="http://minio:9000", + validation_alias=AliasChoices("S3_ENDPOINT_URL", "S3_ENDPOINT"), + ) + s3_public_endpoint_url: str = Field( + default="http://localhost:9000", + validation_alias=AliasChoices("S3_PUBLIC_ENDPOINT_URL"), + ) + presigned_upload_ttl_seconds: int = 300 + attachment_cleanup_interval_seconds: int = Field(default=60, ge=5, le=3600) + attachment_cleanup_grace_seconds: int = Field(default=600, ge=0, le=86400) + attachment_cleanup_batch_size: int = Field(default=100, ge=1, le=1000) + s3_region: str = Field(default="us-east-1", validation_alias=AliasChoices("S3_REGION")) + s3_bucket: str = Field(default="messenger-attachments", validation_alias=AliasChoices("S3_BUCKET")) + s3_access_key: str = Field( + default="messenger-api", + validation_alias=AliasChoices("S3_ACCESS_KEY", "S3_ACCESS_KEY_ID"), + ) + s3_secret_key: str = Field( + default="change-me-api-secret", + validation_alias=AliasChoices("S3_SECRET_KEY", "S3_SECRET_ACCESS_KEY"), + ) + + max_upload_bytes: int = 10 * 1024 * 1024 + max_image_pixels: int = 40_000_000 + + # Durable asynchronous image processing. + rabbitmq_url: str = Field( + default="amqp://guest:guest@rabbitmq:5672/", + validation_alias=AliasChoices("RABBITMQ_URL"), + ) + rabbitmq_exchange: str = Field(default="attachments", validation_alias=AliasChoices("RABBITMQ_EXCHANGE")) + rabbitmq_queue: str = Field(default="attachment.process.v1", validation_alias=AliasChoices("RABBITMQ_QUEUE")) + rabbitmq_prefetch: int = Field(default=4, ge=1, le=64) + outbox_poll_interval_seconds: float = Field(default=1.0, ge=0.1, le=30) + outbox_batch_size: int = Field(default=50, ge=1, le=500) + worker_concurrency: int = Field(default=2, ge=1, le=16) + worker_max_attempts: int = Field(default=5, ge=1, le=20) + worker_lease_seconds: int = Field(default=180, ge=30, le=3600) + worker_retry_base_seconds: float = Field(default=2.0, ge=0.1, le=300) + worker_retry_max_seconds: float = Field(default=120.0, ge=1, le=3600) + worker_staging_retention_seconds: int = Field(default=86400, ge=3600) + image_processing_timeout_seconds: int = Field(default=60, ge=1, le=600) + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + ) + + +from app.core.openbao import load_service_secrets + +load_service_secrets() +settings = Settings() diff --git a/app/core/mtls_protocol.py b/app/core/mtls_protocol.py new file mode 100644 index 0000000..ef588c5 --- /dev/null +++ b/app/core/mtls_protocol.py @@ -0,0 +1,52 @@ +import logging + +import uvicorn.protocols.http.h11_impl as h11_impl + + +logger = logging.getLogger("app.mtls") + + +_original_cycle = h11_impl.RequestResponseCycle + + +class MtlsRequestResponseCycle(_original_cycle): + def __init__(self, *args, **kwargs): + scope = kwargs.get("scope") + transport = kwargs.get("transport") + + if scope is None and args: + scope = args[0] + + if transport is None and len(args) > 2: + transport = args[2] + + ssl_object = ( + transport.get_extra_info("ssl_object") + if transport is not None + else None + ) + + peer_certificate = ( + ssl_object.getpeercert() + if ssl_object is not None + else None + ) + + if scope is not None: + scope.setdefault("state", {}) + scope["state"]["tls_client_cert"] = peer_certificate + + sans = (peer_certificate or {}).get("subjectAltName", ()) + + logger.warning( + "mTLS diagnostic: transport=%s ssl_object=%s certificate=%s uri_sans=%s", + transport is not None, + ssl_object is not None, + peer_certificate is not None, + [value for kind, value in sans if kind == "URI"], + ) + + super().__init__(*args, **kwargs) + + +h11_impl.RequestResponseCycle = MtlsRequestResponseCycle \ No newline at end of file diff --git a/app/core/openbao.py b/app/core/openbao.py new file mode 100644 index 0000000..b3d9836 --- /dev/null +++ b/app/core/openbao.py @@ -0,0 +1,49 @@ +"""Fetch narrowly scoped runtime configuration from OpenBao AppRole.""" +import os +from pathlib import Path + +import requests + + +def _read(path_env: str) -> str: + path = os.getenv(path_env, "") + if not path: + raise RuntimeError(f"{path_env} is required when BAO_ENABLED=1") + return Path(path).read_text(encoding="utf-8").strip() + + +def load_service_secrets() -> None: + if os.getenv("BAO_ENABLED", "0").lower() not in {"1", "true", "yes"}: + return + + address = os.environ["BAO_ADDR"].rstrip("/") + ca_file = os.environ["BAO_CACERT"] + role_id = _read("BAO_ROLE_ID_FILE") + secret_id = _read("BAO_SECRET_ID_FILE") + secret_path = os.environ["BAO_SECRET_PATH"].strip("/") + + session = requests.Session() + session.verify = ca_file + response = session.post( + f"{address}/v1/auth/approle/login", + json={"role_id": role_id, "secret_id": secret_id}, + timeout=5, + ) + response.raise_for_status() + token = response.json()["auth"]["client_token"] + + response = session.get( + f"{address}/v1/secret/data/{secret_path}", + headers={"X-Vault-Token": token}, + timeout=5, + ) + response.raise_for_status() + values = response.json()["data"]["data"] + + allowed = { + "DATABASE_URL", "CACHE_URL", "S3_ENDPOINT", "S3_ACCESS_KEY", + "S3_SECRET_KEY", "S3_PUBLIC_ENDPOINT_URL", "RABBITMQ_URL", "OTEL_EXPORTER_OTLP_ENDPOINT", + } + for key, value in values.items(): + if key in allowed and isinstance(value, str): + os.environ[key] = value diff --git a/app/core/telemetry.py b/app/core/telemetry.py new file mode 100644 index 0000000..1545520 --- /dev/null +++ b/app/core/telemetry.py @@ -0,0 +1,121 @@ +import logging +import os +import socket + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, +) +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.logging import LoggingInstrumentor +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +from app.core.config import settings + + +logger = logging.getLogger(__name__) + +_initialized = False + + +def setup_telemetry(app, engine) -> None: + """Initialize OpenTelemetry instrumentation for the API service.""" + global _initialized + + if _initialized: + return + + service_name = os.getenv("OTEL_SERVICE_NAME", "messenger-api") + endpoint = os.getenv( + "OTEL_EXPORTER_OTLP_ENDPOINT", + "http://otel-collector:4318", + ).rstrip("/") + + resource = Resource.create( + { + "service.name": service_name, + "service.version": "1.4.0", + "service.instance.id": settings.instance_id or socket.gethostname(), + "deployment.environment.name": settings.app_env, + } + ) + + # Distributed tracing. + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter( + endpoint=f"{endpoint}/v1/traces", + certificate_file=os.getenv("TLS_CA_FILE") or None, + ) + ) + ) + trace.set_tracer_provider(tracer_provider) + + # Metrics. + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter( + endpoint=f"{endpoint}/v1/metrics", + certificate_file=os.getenv("TLS_CA_FILE") or None, + ), + export_interval_millis=10000, + ) + meter_provider = MeterProvider( + resource=resource, + metric_readers=[metric_reader], + ) + metrics.set_meter_provider(meter_provider) + + # Logs. + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter( + endpoint=f"{endpoint}/v1/logs", + certificate_file=os.getenv("TLS_CA_FILE") or None, + ) + ) + ) + set_logger_provider(logger_provider) + + # Add trace_id and span_id to Python log records. + LoggingInstrumentor().instrument( + tracer_provider=tracer_provider, + set_logging_format=False, + ) + + # Export Python logging records through OTLP. + logging_handler = LoggingHandler( + level=logging.INFO, + logger_provider=logger_provider, + ) + logging.getLogger().addHandler(logging_handler) + + # FastAPI HTTP server spans. Health endpoints are excluded. + FastAPIInstrumentor.instrument_app( + app, + tracer_provider=tracer_provider, + meter_provider=meter_provider, + excluded_urls=r".*/health/(live|ready)$", + ) + + # SQLAlchemy database spans. + SQLAlchemyInstrumentor().instrument( + engine=engine, + tracer_provider=tracer_provider, + ) + + _initialized = True + logger.info("OpenTelemetry initialized for service %s", service_name) diff --git a/app/core/tls.py b/app/core/tls.py new file mode 100644 index 0000000..b6c539f --- /dev/null +++ b/app/core/tls.py @@ -0,0 +1,7 @@ +import os +import ssl + + +def service_ssl_context() -> ssl.SSLContext: + ca_file = os.getenv("TLS_CA_FILE", "/run/tls/ca.crt") + return ssl.create_default_context(cafile=ca_file) diff --git a/app/core/worker_telemetry.py b/app/core/worker_telemetry.py new file mode 100644 index 0000000..e1640bf --- /dev/null +++ b/app/core/worker_telemetry.py @@ -0,0 +1,242 @@ +import logging +import os +import socket +import time + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, +) +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.instrumentation.logging import LoggingInstrumentor +from opentelemetry.propagate import extract +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.trace import SpanKind, Status, StatusCode + + +logger = logging.getLogger(__name__) + +_initialized = False + +tracer = None +jobs_processed = None +jobs_failed = None +job_duration = None + + +def setup_worker_telemetry(): + """Initialize OpenTelemetry for the background worker.""" + global _initialized + global tracer, jobs_processed, jobs_failed, job_duration + + if _initialized: + return + + endpoint = os.getenv( + "OTEL_EXPORTER_OTLP_ENDPOINT", + "http://otel-collector:4318", + ).rstrip("/") + + service_name = os.getenv( + "OTEL_SERVICE_NAME", + "messenger-worker", + ) + + resource = Resource.create( + { + "service.name": service_name, + "service.version": "1.4.0", + "service.instance.id": ( + os.getenv("INSTANCE_ID") + or socket.gethostname() + ), + "deployment.environment.name": os.getenv( + "APP_ENV", + "lab", + ), + } + ) + + # Distributed tracing. + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter( + endpoint=f"{endpoint}/v1/traces", + certificate_file=os.getenv("TLS_CA_FILE") or None, + ) + ) + ) + trace.set_tracer_provider(tracer_provider) + + # Metrics. + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter( + endpoint=f"{endpoint}/v1/metrics", + certificate_file=os.getenv("TLS_CA_FILE") or None, + ), + export_interval_millis=10000, + ) + + meter_provider = MeterProvider( + resource=resource, + metric_readers=[metric_reader], + ) + metrics.set_meter_provider(meter_provider) + + tracer = trace.get_tracer("messenger.worker") + meter = metrics.get_meter("messenger.worker") + + jobs_processed = meter.create_counter( + name="jobs_processed", + description="Number of successfully processed background jobs", + unit="1", + ) + + jobs_failed = meter.create_counter( + name="jobs_failed", + description="Number of failed background jobs", + unit="1", + ) + + job_duration = meter.create_histogram( + name="job_processing_duration", + description="Background job processing duration", + unit="s", + ) + + # Logs. + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter( + endpoint=f"{endpoint}/v1/logs", + certificate_file=os.getenv("TLS_CA_FILE") or None, + ) + ) + ) + set_logger_provider(logger_provider) + + # Add trace_id and span_id to Python log records. + LoggingInstrumentor().instrument( + tracer_provider=tracer_provider, + set_logging_format=False, + ) + + # Export Python logging records through OTLP. + logging_handler = LoggingHandler( + level=logging.INFO, + logger_provider=logger_provider, + ) + logging.getLogger().addHandler(logging_handler) + + _initialized = True + + logger.info( + "OpenTelemetry initialized for worker service %s", + service_name, + ) + + +def process_job_with_telemetry(process_function, payload): + """Run one job inside a consumer span and record its metrics.""" + if not _initialized: + setup_worker_telemetry() + + started = time.perf_counter() + outcome = "other" + + # Continue the trace started by the API request. + context = extract(payload.get("trace_context", {})) + + with tracer.start_as_current_span( + "worker.process_job", + context=context, + kind=SpanKind.CONSUMER, + attributes={ + "messaging.system": "rabbitmq", + "messaging.operation": "process", + "job.type": "image", + }, + ) as span: + try: + result = process_function(payload) + + state = result.get("state", "unknown") + span.set_attribute("job.outcome", state) + + if state == "ready": + outcome = "success" + jobs_processed.add( + 1, + { + "job_type": "image", + "outcome": "success", + }, + ) + + elif state == "failed": + outcome = "failed" + span.set_status( + Status( + StatusCode.ERROR, + "Background job processing failed", + ) + ) + jobs_failed.add( + 1, + { + "job_type": "image", + "outcome": "failed", + }, + ) + + elif state in {"retry", "busy", "deferred"}: + outcome = "retry" + + else: + outcome = "other" + + return result + + except Exception as exc: + outcome = "failed" + span.set_attribute("job.outcome", "failed") + span.record_exception(exc) + span.set_status( + Status( + StatusCode.ERROR, + "Background job processing failed", + ) + ) + + jobs_failed.add( + 1, + { + "job_type": "image", + "outcome": "failed", + }, + ) + + raise + + finally: + elapsed = time.perf_counter() - started + + job_duration.record( + elapsed, + { + "job_type": "image", + "outcome": outcome, + }, + ) diff --git a/app/db/__init__.py b/app/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/db/base.py b/app/db/base.py new file mode 100644 index 0000000..1c2dcc4 --- /dev/null +++ b/app/db/base.py @@ -0,0 +1,5 @@ +from sqlalchemy.orm import DeclarativeBase + + +class Base(DeclarativeBase): + pass \ No newline at end of file diff --git a/app/db/models.py b/app/db/models.py new file mode 100644 index 0000000..385815c --- /dev/null +++ b/app/db/models.py @@ -0,0 +1,488 @@ +import uuid +from datetime import datetime, timezone +from sqlalchemy import ( + DateTime, + JSON, + ForeignKey, + Index, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship +from app.db.base import Base + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + +class User(Base): + __tablename__ = "users" + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + ) + username: Mapped[str] = mapped_column( + String(32), + unique=True, + nullable=False, + index=True, + ) + display_name: Mapped[str] = mapped_column( + String(100), + nullable=False, + ) + password_hash: Mapped[str] = mapped_column( + String(255), + nullable=False, + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utc_now, + nullable=False, + ) + memberships: Mapped[list["ChatMember"]] = relationship( + back_populates="user", + cascade="all, delete-orphan", + ) + messages: Mapped[list["Message"]] = relationship( + back_populates="sender", + ) + +class Chat(Base): + __tablename__ = "chats" + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + ) + title: Mapped[str] = mapped_column( + String(100), + nullable=False, + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utc_now, + nullable=False, + ) + members: Mapped[list["ChatMember"]] = relationship( + back_populates="chat", + cascade="all, delete-orphan", + ) + messages: Mapped[list["Message"]] = relationship( + back_populates="chat", + cascade="all, delete-orphan", + ) + +class ChatMember(Base): + __tablename__ = "chat_members" + chat_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("chats.id", ondelete="CASCADE"), + primary_key=True, + ) + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + primary_key=True, + ) + chat: Mapped["Chat"] = relationship( + back_populates="members", + ) + user: Mapped["User"] = relationship( + back_populates="memberships", + ) + +class Message(Base): + __tablename__ = "messages" + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + ) + chat_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("chats.id", ondelete="CASCADE"), + nullable=False, + ) + sender_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id"), + nullable=False, + ) + text: Mapped[str] = mapped_column( + Text, + nullable=False, + ) + attachment_ids: Mapped[list] = mapped_column(JSON, nullable=False, default=list) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utc_now, + nullable=False, + ) + chat: Mapped["Chat"] = relationship( + back_populates="messages", + ) + sender: Mapped["User"] = relationship( + back_populates="messages", + ) + __table_args__ = ( + Index( + "ix_messages_chat_created_id", + "chat_id", + "created_at", + "id", + ), + ) + +class Session(Base): + __tablename__ = "sessions" + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + ) + token_hash: Mapped[str] = mapped_column( + String(64), + unique=True, + nullable=False, + index=True, + ) + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ) + expires_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utc_now, + nullable=False, + ) + user: Mapped["User"] = relationship() + +class IdempotencyRecord(Base): + __tablename__ = "idempotency_records" + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ) + method: Mapped[str] = mapped_column(String(10), nullable=False) + path: Mapped[str] = mapped_column(String(255), nullable=False) + idempotency_key: Mapped[str] = mapped_column(String(128), nullable=False) + request_hash: Mapped[str] = mapped_column(String(64), nullable=False) + # NULL is visible only inside the creating transaction; the message and + # idempotency row are committed atomically. + message_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), nullable=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utc_now, nullable=False + ) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + __table_args__ = ( + UniqueConstraint( + "user_id", "method", "path", "idempotency_key", + name="uq_idempotency_scope_key", + ), + Index("ix_idempotency_records_expires_at", "expires_at"), + ) + +class Attachment(Base): + __tablename__ = "attachments" + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + ) + chat_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("chats.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + owner_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ) + filename: Mapped[str] = mapped_column(String(255), nullable=False) + content_type: Mapped[str] = mapped_column(String(32), nullable=False) + size_bytes: Mapped[int] = mapped_column(nullable=False) + sha256: Mapped[str] = mapped_column(String(64), nullable=False) + # Ключ оригинала в конечном S3-prefix. + object_key: Mapped[str] = mapped_column( + String(512), + unique=True, + nullable=False, + ) + # Ключ исходного файла в durable staging. + staging_key: Mapped[str | None] = mapped_column( + String(512), + nullable=True, + ) + status: Mapped[str] = mapped_column( + String(16), + nullable=False, + default="ready", + ) + error_code: Mapped[str | None] = mapped_column( + String(64), + nullable=True, + ) + pipeline_version: Mapped[int] = mapped_column( + nullable=False, + default=1, + server_default="1", ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utc_now, + nullable=False, + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utc_now, + onupdate=utc_now, + nullable=False, + ) + __table_args__ = ( + UniqueConstraint( + "staging_key", + name="uq_attachments_staging_key", + ), + ) + +class AttachmentUpload(Base): + __tablename__ = "attachment_uploads" + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + ) + chat_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("chats.id", ondelete="CASCADE"), + nullable=False, + ) + owner_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ) + filename: Mapped[str] = mapped_column( + String(255), + nullable=False, + ) + content_type: Mapped[str] = mapped_column( + String(32), + nullable=False, + ) + expected_size_bytes: Mapped[int] = mapped_column( + nullable=False, + ) + expected_sha256: Mapped[str] = mapped_column( + String(64), + nullable=False, + ) + object_key: Mapped[str] = mapped_column( + String(512), + unique=True, + nullable=False, + ) + status: Mapped[str] = mapped_column( + String(16), + nullable=False, + default="pending", + ) + expires_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utc_now, + nullable=False, + ) + completed_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), + nullable=True, + ) + __table_args__ = ( + Index("ix_attachment_uploads_chat_id", "chat_id"), + Index("ix_attachment_uploads_owner_id", "owner_id"), + Index("ix_attachment_uploads_expires_at", "expires_at"), + ) + +class ProcessingJob(Base): + __tablename__ = "processing_jobs" + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + ) + attachment_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("attachments.id", ondelete="CASCADE"), + nullable=False, + ) + pipeline_version: Mapped[int] = mapped_column( + nullable=False, + default=1, + server_default="1", ) + attempt: Mapped[int] = mapped_column( + nullable=False, + default=1, + server_default="1", ) + status: Mapped[str] = mapped_column( + String(16), + nullable=False, + default="queued", + server_default="queued", ) + available_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utc_now, + nullable=False, + ) + lease_until: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), + nullable=True, + ) + worker_id: Mapped[str | None] = mapped_column( + String(128), + nullable=True, + ) + last_error_code: Mapped[str | None] = mapped_column( + String(64), + nullable=True, + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utc_now, + nullable=False, + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utc_now, + onupdate=utc_now, + nullable=False, + ) + __table_args__ = ( + UniqueConstraint( + "attachment_id", + "pipeline_version", + name="uq_processing_job_attachment_pipeline", + ), + Index( + "ix_processing_jobs_status_available", + "status", + "available_at", + ), + Index( + "ix_processing_jobs_lease_until", + "lease_until", + ), + ) + +class AttachmentVariant(Base): + __tablename__ = "attachment_variants" + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + ) + attachment_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("attachments.id", ondelete="CASCADE"), + nullable=False, + ) + variant: Mapped[str] = mapped_column( + String(32), + nullable=False, + ) + object_key: Mapped[str] = mapped_column( + String(512), + unique=True, + nullable=False, + ) + content_type: Mapped[str] = mapped_column( + String(32), + nullable=False, + ) + width: Mapped[int] = mapped_column(nullable=False) + height: Mapped[int] = mapped_column(nullable=False) + size_bytes: Mapped[int] = mapped_column(nullable=False) + sha256: Mapped[str] = mapped_column( + String(64), + nullable=False, + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utc_now, + nullable=False, + ) + __table_args__ = ( + UniqueConstraint( + "attachment_id", + "variant", + name="uq_attachment_variant", + ), + Index( + "ix_attachment_variants_attachment_id", + "attachment_id", + ), + ) + +class OutboxEvent(Base): + __tablename__ = "outbox_events" + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + ) + event_type: Mapped[str] = mapped_column( + String(128), + nullable=False, + ) + aggregate_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + nullable=False, + ) + payload: Mapped[dict] = mapped_column( + JSON, + nullable=False, + ) + status: Mapped[str] = mapped_column( + String(16), + nullable=False, + default="pending", + server_default="pending", ) + attempts: Mapped[int] = mapped_column( + nullable=False, + default=0, + server_default="0", ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utc_now, + nullable=False, + ) + published_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), + nullable=True, + ) + last_error: Mapped[str | None] = mapped_column( + String(256), + nullable=True, + ) + __table_args__ = ( + Index( + "ix_outbox_events_status_created", + "status", + "created_at", + ), + ) diff --git a/app/db/session.py b/app/db/session.py new file mode 100644 index 0000000..190589b --- /dev/null +++ b/app/db/session.py @@ -0,0 +1,15 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from app.core.config import settings + +engine = create_engine( + settings.database_url, + pool_pre_ping=True, + pool_size=settings.db_pool_size, + max_overflow=settings.db_max_overflow, + pool_timeout=settings.db_pool_timeout, + pool_recycle=1800, +) + +SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False) diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..b9d363b --- /dev/null +++ b/app/main.py @@ -0,0 +1,250 @@ +import asyncio +import logging +import time +import uuid +from contextlib import asynccontextmanager, suppress +from pathlib import Path + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from sqlalchemy import text + +from app.api.auth import router as auth_router +from app.api.users import router as users_router +from app.api.chats import router as chats_router +from app.api.messages import router as messages_router +from app.api.attachments import router as attachments_router +import app.core.mtls_protocol +from app.core.config import settings +from app.core.telemetry import setup_telemetry +from app.db.session import engine +from app.services.object_storage import client as s3_client +from app.services.attachment_cleanup import cleanup_expired_uploads + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", +) + +logger = logging.getLogger(__name__) + + +async def attachment_cleanup_loop(): + while True: + try: + # Synchronous boto3/SQLAlchemy I/O must not block the event loop. + await asyncio.to_thread(cleanup_expired_uploads) + except Exception: + logger.exception("Attachment cleanup worker iteration failed") + + await asyncio.sleep(settings.attachment_cleanup_interval_seconds) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + logger.info("Application startup instance_id=%s", settings.instance_id) + + cleanup_task = asyncio.create_task(attachment_cleanup_loop()) + + try: + yield + finally: + cleanup_task.cancel() + + with suppress(asyncio.CancelledError): + await cleanup_task + + engine.dispose() + logger.info("Application shutdown instance_id=%s", settings.instance_id) + + +app = FastAPI( + title=settings.app_name, + version="1.4.0", + lifespan=lifespan, +) + +app.include_router(users_router) +app.include_router(auth_router) +app.include_router(chats_router) +app.include_router(messages_router) +app.include_router(attachments_router) + + +@app.middleware("http") +async def response_headers(request: Request, call_next): + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + + # API only accepts requests carrying the fixed identity of the private + # ingress. The TLS listener additionally requires a client certificate + # issued by the dedicated ingress CA. + peer_cert = getattr(request.state, "tls_client_cert", None) + + uri_sans = [ + value + for kind, value in (peer_cert or {}).get("subjectAltName", ()) + if kind == "URI" + ] + + if "spiffe://lab7/ingress/nginx" not in uri_sans: + return JSONResponse( + status_code=403, + content={"detail": "Untrusted client certificate identity"}, + ) + + started_at = time.perf_counter() + + try: + response = await call_next(request) + + except Exception: + logger.exception( + "Unhandled request error; request_id=%s", + request_id, + ) + + response = JSONResponse( + status_code=500, + content={ + "error": { + "code": "internal_error", + "message": "Unexpected error", + "request_id": request_id, + } + }, + ) + + duration_ms = (time.perf_counter() - started_at) * 1000 + + route = request.scope.get("route") + route_template = getattr(route, "path", "unmatched") + + if route_template not in ("/health/live", "/health/ready"): + logger.info( + "HTTP request completed", + extra={ + "event": "http_request_completed", + "http_method": request.method, + "http_route": route_template, + "http_status_code": response.status_code, + "http_duration_ms": round(duration_ms, 3), + }, + ) + + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = settings.instance_id + response.headers["X-Content-Type-Options"] = "nosniff" + + return response + + +# Initialize OpenTelemetry AFTER registering the application middleware. +# This makes OpenTelemetry middleware outermost, keeping the server span +# active while response_headers writes the completed-request log. +setup_telemetry(app, engine) + + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + request_id = getattr( + request.state, + "request_id", + str(uuid.uuid4()), + ) + + code = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 500: "internal_error", + 503: "unavailable", + }.get(exc.status_code, "internal_error") + + headers = dict(exc.headers or {}) + headers["X-Request-Id"] = request_id + headers["X-Instance-Id"] = settings.instance_id + + return JSONResponse( + status_code=exc.status_code, + content={ + "error": { + "code": code, + "message": ( + exc.detail + if isinstance(exc.detail, str) + else "Request failed" + ), + "request_id": request_id, + } + }, + headers=headers, + ) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler( + request: Request, + exc: RequestValidationError, +): + request_id = getattr( + request.state, + "request_id", + str(uuid.uuid4()), + ) + + return JSONResponse( + status_code=400, + content={ + "error": { + "code": "invalid_request", + "message": "Validation error", + "request_id": request_id, + } + }, + headers={ + "X-Request-Id": request_id, + "X-Instance-Id": settings.instance_id, + }, + ) + + +@app.get("/health/live") +def liveness(): + return {"status": "ok"} + + +DRAIN_FILE = Path("/tmp/api-draining") + + +@app.get("/health/ready") +def readiness(): + if DRAIN_FILE.exists(): + raise HTTPException( + status_code=503, + detail="Instance is draining", + ) + + try: + with engine.connect() as connection: + connection.execute(text("SELECT 1")) + + s3_client().head_bucket(Bucket=settings.s3_bucket) + + return {"status": "ok"} + + except Exception: + logger.exception( + "PostgreSQL or S3 readiness check failed" + ) + + raise HTTPException( + status_code=503, + detail="Required storage unavailable", + ) from None diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..68199a0 --- /dev/null +++ b/app/models.py @@ -0,0 +1,138 @@ + +from datetime import datetime, timezone +from typing import List, Optional, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class BaseConfig(BaseModel): + model_config = ConfigDict( + from_attributes=True, + json_encoders={ + datetime: lambda value: ( + value.astimezone(timezone.utc) + .isoformat(timespec="microseconds") + .replace("+00:00", "Z") + ) + }, + ) + + +class CreateUser(BaseConfig): + username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(..., min_length=1, max_length=100) + password: str = Field(..., min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def validate_display_name(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("display_name must not be blank") + return value + + +class User(BaseConfig): + id: str + username: str + display_name: str + created_at: datetime + + +class CreateChat(BaseConfig): + title: str = Field(..., min_length=1, max_length=100) + member_ids: List[str] = Field(..., min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def validate_title(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("title must not be blank") + return value + + @field_validator("member_ids") + @classmethod + def validate_unique_members(cls, value: List[str]) -> List[str]: + if len(value) != len(set(value)): + raise ValueError("member_ids must be unique") + return value + + +class Chat(BaseConfig): + id: str + title: str + member_ids: List[str] + created_at: datetime + + +class CreateMessage(BaseConfig): + text: str = Field(..., max_length=4000) + attachment_ids: List[str] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def validate_attachment_ids(cls, value): + if len(value) != len(set(value)): + raise ValueError("attachment_ids must be unique") + return value + + @field_validator("text") + @classmethod + def validate_text(cls, value: str) -> str: + value = value.strip() + return value + + +class Message(BaseConfig): + id: str + chat_id: str + sender_id: str + text: str + created_at: datetime + attachment_ids: List[str] = Field(default_factory=list) + + +class Health(BaseConfig): + status: str = "ok" + + +class ErrorDetail(BaseConfig): + code: str + message: str + request_id: str + + +class ErrorResponse(BaseConfig): + error: ErrorDetail + + +class ChatPage(BaseConfig): + items: List[Chat] + next_cursor: Optional[str] = None + + +class MessagePage(BaseConfig): + items: List[Message] + next_cursor: Optional[str] = None + + +class CreateAttachmentUpload(BaseConfig): + filename: str = Field(..., min_length=1, max_length=255) + content_type: Literal["image/png", "image/jpeg"] + size_bytes: int = Field(..., gt=0, le=10 * 1024 * 1024) + sha256: str = Field(..., pattern=r"^[a-fA-F0-9]{64}$") + + @field_validator("filename") + @classmethod + def validate_filename(cls, value: str) -> str: + value = value.strip() + + if ( + not value + or any(c in value for c in ("/", "\\", "\x00")) + or any(ord(c) < 32 or ord(c) == 127 for c in value) + ): + raise ValueError("Unsafe filename") + + return value \ No newline at end of file diff --git a/app/repositories/__init__.py b/app/repositories/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/repositories/chat_repository.py b/app/repositories/chat_repository.py new file mode 100644 index 0000000..e5cd838 --- /dev/null +++ b/app/repositories/chat_repository.py @@ -0,0 +1,71 @@ +import uuid + +from sqlalchemy import select +from sqlalchemy.orm import Session, selectinload + +from app.db.models import Chat, ChatMember + + +class ChatRepository: + + def __init__(self, db: Session): + self.db = db + + def get_by_id(self, chat_id: uuid.UUID) -> Chat | None: + statement = ( + select(Chat) + .options(selectinload(Chat.members)) + .where(Chat.id == chat_id) + ) + + return self.db.scalar(statement) + + def get_by_user( + self, + user_id: uuid.UUID, + limit: int = 50, + ) -> list[Chat]: + statement = ( + select(Chat) + .join(ChatMember, ChatMember.chat_id == Chat.id) + .where(ChatMember.user_id == user_id) + .order_by(Chat.created_at.asc(), Chat.id.asc()) + .limit(limit) + ) + + return list(self.db.scalars(statement).all()) + + def is_member( + self, + chat_id: uuid.UUID, + user_id: uuid.UUID, + ) -> bool: + statement = select(ChatMember).where( + ChatMember.chat_id == chat_id, + ChatMember.user_id == user_id, + ) + + return self.db.scalar(statement) is not None + + def create( + self, + title: str, + member_ids: list[uuid.UUID], + ) -> Chat: + chat = Chat(title=title) + + self.db.add(chat) + self.db.flush() + + memberships = [ + ChatMember( + chat_id=chat.id, + user_id=user_id, + ) + for user_id in member_ids + ] + + self.db.add_all(memberships) + self.db.flush() + + return chat \ No newline at end of file diff --git a/app/repositories/message_repository.py b/app/repositories/message_repository.py new file mode 100644 index 0000000..85b25da --- /dev/null +++ b/app/repositories/message_repository.py @@ -0,0 +1,73 @@ +import uuid + +from datetime import datetime + +from sqlalchemy import and_, or_, select +from sqlalchemy.orm import Session + +from app.db.models import Message + + +class MessageRepository: + + def __init__(self, db: Session): + self.db = db + + def get_by_id( + self, + message_id: uuid.UUID, + ) -> Message | None: + statement = select(Message).where( + Message.id == message_id + ) + + return self.db.scalar(statement) + + def create( + self, + chat_id: uuid.UUID, + sender_id: uuid.UUID, + text: str, + attachment_ids: list[str] | None = None, + ) -> Message: + message = Message( + chat_id=chat_id, + sender_id=sender_id, + text=text, + attachment_ids=attachment_ids or [], + ) + + self.db.add(message) + self.db.flush() + + return message + + def get_history( + self, + chat_id: uuid.UUID, + limit: int = 50, + cursor_created_at: datetime | None = None, + cursor_id: uuid.UUID | None = None, + ) -> list[Message]: + statement = select(Message).where( + Message.chat_id == chat_id + ) + + if cursor_created_at is not None and cursor_id is not None: + statement = statement.where( + or_( + Message.created_at > cursor_created_at, + and_( + Message.created_at == cursor_created_at, + Message.id > cursor_id, + ), + ) + ) + + statement = ( + statement + .order_by(Message.created_at.asc(), Message.id.asc()) + .limit(limit) + ) + + return list(self.db.scalars(statement).all()) \ No newline at end of file diff --git a/app/repositories/session_repository.py b/app/repositories/session_repository.py new file mode 100644 index 0000000..7366b17 --- /dev/null +++ b/app/repositories/session_repository.py @@ -0,0 +1,78 @@ +import uuid + +from datetime import datetime, timezone + +from sqlalchemy import delete, select +from sqlalchemy.orm import Session + +from app.db.models import Session as UserSession + + +class SessionRepository: + + def __init__(self, db: Session): + self.db = db + + def create( + self, + user_id: uuid.UUID, + token_hash: str, + expires_at: datetime, + ) -> UserSession: + session = UserSession( + user_id=user_id, + token_hash=token_hash, + expires_at=expires_at, + ) + + self.db.add(session) + self.db.flush() + + return session + + def get_by_token_hash( + self, + token_hash: str, + ) -> UserSession | None: + statement = select(UserSession).where( + UserSession.token_hash == token_hash + ) + + return self.db.scalar(statement) + + def get_valid_session( + self, + token_hash: str, + ) -> UserSession | None: + now = datetime.now(timezone.utc) + + statement = select(UserSession).where( + UserSession.token_hash == token_hash, + UserSession.expires_at > now, + ) + + return self.db.scalar(statement) + + def delete_by_token_hash( + self, + token_hash: str, + ) -> bool: + statement = ( + delete(UserSession) + .where(UserSession.token_hash == token_hash) + .returning(UserSession.id) + ) + + return self.db.scalar(statement) is not None + + def delete_expired(self) -> int: + now = datetime.now(timezone.utc) + + statement = ( + delete(UserSession) + .where(UserSession.expires_at <= now) + ) + + result = self.db.execute(statement) + + return result.rowcount \ No newline at end of file diff --git a/app/repositories/user_repository.py b/app/repositories/user_repository.py new file mode 100644 index 0000000..4569dcd --- /dev/null +++ b/app/repositories/user_repository.py @@ -0,0 +1,41 @@ +import uuid + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.db.models import User + + +class UserRepository: + + def __init__(self, db: Session): + self.db = db + + def get_by_id(self, user_id: uuid.UUID) -> User | None: + statement = select(User).where(User.id == user_id) + + return self.db.scalar(statement) + + def get_by_username(self, username: str) -> User | None: + statement = select(User).where( + User.username == username + ) + + return self.db.scalar(statement) + + def create( + self, + username: str, + display_name: str, + password_hash: str, + ) -> User: + user = User( + username=username, + display_name=display_name, + password_hash=password_hash, + ) + + self.db.add(user) + self.db.flush() + + return user \ No newline at end of file diff --git a/app/services/__init__.py b/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/attachment_cleanup.py b/app/services/attachment_cleanup.py new file mode 100644 index 0000000..c56e4ce --- /dev/null +++ b/app/services/attachment_cleanup.py @@ -0,0 +1,66 @@ +"""Best-effort cleanup for abandoned presigned uploads.""" + +import logging +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select + +from app.core.config import settings +from app.db.models import AttachmentUpload +from app.db.session import SessionLocal +from app.services.object_storage import client + +logger = logging.getLogger(__name__) + + +def cleanup_expired_uploads() -> int: + """Delete old pending objects and mark their upload records expired. + + A grace period protects clients completing an upload near URL expiry. + PostgreSQL SKIP LOCKED makes this safe across multiple API replicas. + """ + cutoff = datetime.now(timezone.utc) - timedelta( + seconds=settings.attachment_cleanup_grace_seconds + ) + deleted_count = 0 + + with SessionLocal() as db: + uploads = db.scalars( + select(AttachmentUpload) + .where( + AttachmentUpload.status == "pending", + AttachmentUpload.expires_at <= cutoff, + ) + .order_by(AttachmentUpload.expires_at.asc()) + .limit(settings.attachment_cleanup_batch_size) + .with_for_update(skip_locked=True) + ).all() + + if not uploads: + return 0 + + s3 = client() + try: + for upload in uploads: + # DeleteObject is idempotent: a missing object is also safe. + s3.delete_object( + Bucket=settings.s3_bucket, + Key=upload.object_key, + ) + upload.status = "expired" + deleted_count += 1 + + db.commit() + except Exception: + db.rollback() + logger.exception( + "Attachment cleanup batch failed; pending records will be retried" + ) + return 0 + + if deleted_count: + logger.info( + "Expired abandoned attachment uploads cleaned: count=%d", + deleted_count, + ) + return deleted_count diff --git a/app/services/auth_service.py b/app/services/auth_service.py new file mode 100644 index 0000000..f17d133 --- /dev/null +++ b/app/services/auth_service.py @@ -0,0 +1,233 @@ + +import hashlib +import secrets +from datetime import datetime, timedelta, timezone + +from redis import Redis +from redis.backoff import NoBackoff +from redis.exceptions import RedisError +from redis.retry import Retry +from sqlalchemy.orm import Session + +from app.core.config import settings +from app.db.models import User +from app.repositories.session_repository import SessionRepository +from app.repositories.user_repository import UserRepository +from app.services.user_service import password_hasher + + +class InvalidCredentialsError(Exception): + pass + + +class InvalidSessionError(Exception): + pass + + +class LoginRateLimitError(Exception): + def __init__(self, retry_after: int): + self.retry_after = max(1, retry_after) + super().__init__("Too many login attempts") + + +class LoginRateLimitUnavailableError(Exception): + pass + + +class AuthService: + def __init__(self, db: Session): + self.db = db + self.sessions = SessionRepository(db) + self.users = UserRepository(db) + self._redis_available = True + + self.redis = Redis.from_url( + settings.cache_url, + decode_responses=True, + socket_connect_timeout=0.5, + socket_timeout=0.5, + retry_on_timeout=False, + retry=Retry(NoBackoff(), 0), + ) + + @staticmethod + def _hash_token(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + @staticmethod + def _login_rate_key(username: str, client_ip: str) -> str: + identity = hashlib.sha256( + f"{username.strip().lower()}|{client_ip}".encode("utf-8") + ).hexdigest() + return f"login-rate:{identity}" + + def _check_login_rate(self, key: str) -> None: + try: + count = int(self.redis.get(key) or 0) + + if count >= settings.login_rate_limit_attempts: + ttl = self.redis.ttl(key) + raise LoginRateLimitError( + ttl if ttl > 0 else settings.login_rate_limit_window_seconds + ) + except LoginRateLimitError: + raise + except RedisError: + raise LoginRateLimitUnavailableError() from None + + def _record_failed_login(self, key: str) -> None: + script = """ + local count = redis.call('INCR', KEYS[1]) + if count == 1 then + redis.call('EXPIRE', KEYS[1], ARGV[1]) + end + local ttl = redis.call('TTL', KEYS[1]) + return {count, ttl} + """ + + try: + count, ttl = self.redis.eval( + script, + 1, + key, + settings.login_rate_limit_window_seconds, + ) + except RedisError: + raise LoginRateLimitUnavailableError() from None + + if int(count) > settings.login_rate_limit_attempts: + raise LoginRateLimitError( + int(ttl) if int(ttl) > 0 + else settings.login_rate_limit_window_seconds + ) + + def create_session( + self, + username: str, + password: str, + client_ip: str = "unknown", + ) -> dict: + rate_key = self._login_rate_key(username, client_ip) + + self._check_login_rate(rate_key) + + user = self.users.get_by_username(username) + + if user is None or not password_hasher.verify( + password, user.password_hash + ): + self._record_failed_login(rate_key) + raise InvalidCredentialsError( + "Invalid username or password" + ) + + try: + self.redis.delete(rate_key) + except RedisError: + raise LoginRateLimitUnavailableError() from None + + token = secrets.token_urlsafe(32) + token_hash = self._hash_token(token) + + expires_at = datetime.now(timezone.utc) + timedelta( + seconds=settings.session_ttl_seconds + ) + + try: + self.sessions.create( + user_id=user.id, + token_hash=token_hash, + expires_at=expires_at, + ) + self.db.commit() + except Exception: + self.db.rollback() + raise + + self._cache_session(token_hash, user.id, expires_at) + + return { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at, + "user": user, + } + + def _cache_session( + self, + token_hash: str, + user_id, + expires_at, + ) -> None: + if not self._redis_available: + return + + ttl = max( + 1, + int((expires_at - datetime.now(timezone.utc)).total_seconds()), + ) + + try: + self.redis.setex( + f"session:{token_hash}", + ttl, + str(user_id), + ) + except RedisError: + self._redis_available = False + + def authenticate(self, token: str) -> User: + token_hash = self._hash_token(token) + cached_user_id = None + + if self._redis_available: + try: + cached_user_id = self.redis.get( + f"session:{token_hash}" + ) + except RedisError: + self._redis_available = False + + session = self.sessions.get_valid_session(token_hash) + + if session is None: + if self._redis_available: + try: + self.redis.delete(f"session:{token_hash}") + except RedisError: + self._redis_available = False + + raise InvalidSessionError( + "Session is invalid or expired" + ) + + if cached_user_id is None: + self._cache_session( + token_hash, + session.user_id, + session.expires_at, + ) + + return session.user + + def delete_session(self, token: str) -> None: + token_hash = self._hash_token(token) + + try: + deleted = self.sessions.delete_by_token_hash(token_hash) + + if not deleted: + raise InvalidSessionError( + "Session is invalid or expired" + ) + + self.db.commit() + except Exception: + self.db.rollback() + raise + + if self._redis_available: + try: + self.redis.delete(f"session:{token_hash}") + except RedisError: + self._redis_available = False diff --git a/app/services/chat_service.py b/app/services/chat_service.py new file mode 100644 index 0000000..10ff565 --- /dev/null +++ b/app/services/chat_service.py @@ -0,0 +1,94 @@ +import uuid + +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from app.db.models import Chat +from app.repositories.chat_repository import ChatRepository +from app.repositories.user_repository import UserRepository + + +class ChatNotFoundError(Exception): + pass + + +class ChatAccessError(Exception): + pass + + +class InvalidChatDataError(Exception): + pass + + +class ChatService: + def __init__(self, db: Session): + self.db = db + self.chats = ChatRepository(db) + self.users = UserRepository(db) + + def create_chat( + self, + title: str, + member_ids: list[str], + creator_id: uuid.UUID, + ) -> Chat: + title = title.strip() + + if not title or len(title) > 100: + raise InvalidChatDataError("Invalid chat title") + + if not member_ids or len(member_ids) > 100: + raise InvalidChatDataError("Invalid members") + + try: + parsed_ids = [uuid.UUID(str(value)) for value in member_ids] + except (ValueError, TypeError, AttributeError): + raise InvalidChatDataError("Invalid member UUID") from None + + if len(parsed_ids) != len(set(parsed_ids)): + raise InvalidChatDataError("Duplicate members") + + if creator_id not in parsed_ids: + raise InvalidChatDataError("Creator must be a member") + + for user_id in parsed_ids: + if self.users.get_by_id(user_id) is None: + raise InvalidChatDataError("Unknown member") + + try: + chat = self.chats.create( + title=title, + member_ids=parsed_ids, + ) + self.db.commit() + self.db.refresh(chat) + return chat + + except IntegrityError as exc: + self.db.rollback() + raise InvalidChatDataError("Unable to create chat") from exc + + except Exception: + self.db.rollback() + raise + + def get_chat(self, chat_id: uuid.UUID, user_id: uuid.UUID) -> Chat: + chat = self.chats.get_by_id(chat_id) + + if chat is None: + raise ChatNotFoundError() + + if not self.chats.is_member(chat_id, user_id): + raise ChatNotFoundError() + + return chat + + def list_chats(self, user_id: uuid.UUID) -> list[Chat]: + return self.chats.get_by_user(user_id) + + def check_membership( + self, + chat_id: uuid.UUID, + user_id: uuid.UUID, + ) -> bool: + return self.chats.is_member(chat_id, user_id) diff --git a/app/services/image_processor.py b/app/services/image_processor.py new file mode 100644 index 0000000..a9938e2 --- /dev/null +++ b/app/services/image_processor.py @@ -0,0 +1,281 @@ +"""Deterministic image variants for the attachment pipeline.""" + +import hashlib +import io +import multiprocessing +import time +import warnings +from dataclasses import dataclass + +from PIL import Image, ImageOps, UnidentifiedImageError + +from app.core.config import settings + + +@dataclass(frozen=True) +class ProcessedVariant: + name: str + content_type: str + extension: str + width: int + height: int + data: bytes + sha256: str + + +class ImageProcessingTimeout(TimeoutError): + """Raised when image processing exceeds its configured deadline.""" + + +def _load_oriented(data: bytes) -> Image.Image: + if len(data) > settings.max_upload_bytes: + raise ValueError("image_too_large") + + Image.MAX_IMAGE_PIXELS = settings.max_image_pixels + + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + + try: + source = Image.open(io.BytesIO(data)) + + if source.format not in {"JPEG", "PNG"}: + raise ValueError("unsupported_image") + + source.verify() + source = Image.open(io.BytesIO(data)) + image = ImageOps.exif_transpose(source) + image.load() + + except ( + UnidentifiedImageError, + OSError, + Image.DecompressionBombError, + Image.DecompressionBombWarning, + ): + raise ValueError("invalid_image") from None + + if image.width * image.height > settings.max_image_pixels: + raise ValueError("image_too_many_pixels") + + return image + + +def _rgb_white(image: Image.Image) -> Image.Image: + if image.mode in ("RGBA", "LA") or ( + image.mode == "P" and "transparency" in image.info + ): + rgba = image.convert("RGBA") + background = Image.new( + "RGBA", + rgba.size, + (255, 255, 255, 255), + ) + return Image.alpha_composite(background, rgba).convert("RGB") + + return image.convert("RGB") + + +def _fit( + image: Image.Image, + box: tuple[int, int], + allow_upscale: bool = False, +) -> Image.Image: + image = image.copy() + image.thumbnail(box, Image.Resampling.LANCZOS) + + if allow_upscale: + scale = min( + box[0] / image.width, + box[1] / image.height, + ) + + if scale > 1: + image = image.resize( + ( + max(1, int(image.width * scale)), + max(1, int(image.height * scale)), + ), + Image.Resampling.LANCZOS, + ) + + return image + + +def _encode( + name: str, + image: Image.Image, + fmt: str, + content_type: str, + extension: str, + **kwargs, +) -> ProcessedVariant: + output = io.BytesIO() + image.save(output, format=fmt, **kwargs) + data = output.getvalue() + + return ProcessedVariant( + name=name, + content_type=content_type, + extension=extension, + width=image.width, + height=image.height, + data=data, + sha256=hashlib.sha256(data).hexdigest(), + ) + + +def create_variants(original: bytes) -> list[ProcessedVariant]: + source = _load_oriented(original) + rgb = _rgb_white(source) + + # 256x256 fit; never upscale; dimensions retain aspect ratio. + thumb = _fit(rgb, (256, 256)) + thumbnail = _encode( + "thumbnail", + thumb, + "JPEG", + "image/jpeg", + "jpg", + quality=85, + optimize=True, + progressive=True, + ) + + # Center crop to square, 128x128. Upscaling is allowed by contract. + side = min(rgb.width, rgb.height) + left = (rgb.width - side) // 2 + top = (rgb.height - side) // 2 + crop = rgb.crop((left, top, left + side, top + side)) + crop = crop.resize((128, 128), Image.Resampling.LANCZOS) + + cropped = _encode( + "crop", + crop, + "JPEG", + "image/jpeg", + "jpg", + quality=85, + optimize=True, + progressive=True, + ) + + # Optimized WebP fit 1280x1280; never upscale. + webp_image = _fit(rgb, (1280, 1280)) + webp = _encode( + "optimized", + webp_image, + "WEBP", + "image/webp", + "webp", + quality=82, + method=6, + ) + + return [thumbnail, cropped, webp] + + +def _variant_process_entry(connection, original: bytes) -> None: + """Run Pillow processing in an isolated child process.""" + + try: + result = create_variants(original) + connection.send(("ok", result)) + + except ValueError as exc: + connection.send(("validation_error", str(exc)[:100])) + + except Exception: + connection.send(("processing_error", "processing_failed")) + + finally: + connection.close() + + +def create_variants_with_timeout( + original: bytes, + timeout_seconds: float, +) -> list[ProcessedVariant]: + """Run image processing in a child process with a hard deadline.""" + + if timeout_seconds <= 0: + raise ValueError("invalid_processing_timeout") + + context = multiprocessing.get_context("spawn") + receive_connection, send_connection = context.Pipe(duplex=False) + + process = context.Process( + target=_variant_process_entry, + args=(send_connection, original), + name="image-processor", + ) + + try: + process.start() + send_connection.close() + + deadline = time.monotonic() + timeout_seconds + message = None + + while time.monotonic() < deadline: + if receive_connection.poll(0.1): + try: + message = receive_connection.recv() + except EOFError: + message = None + break + + if not process.is_alive(): + if receive_connection.poll(): + try: + message = receive_connection.recv() + except EOFError: + message = None + break + + if message is None and process.is_alive(): + process.terminate() + process.join(timeout=2) + + if process.is_alive(): + process.kill() + process.join() + + raise ImageProcessingTimeout( + "image_processing_timeout" + ) + + process.join(timeout=2) + + if process.is_alive(): + process.terminate() + process.join(timeout=2) + + if message is None: + raise RuntimeError("image_processing_failed") + + status, payload = message + + if status == "ok": + return payload + + if status == "validation_error": + raise ValueError(payload) + + raise RuntimeError("image_processing_failed") + + finally: + if process.is_alive(): + process.terminate() + process.join(timeout=2) + + if process.is_alive(): + process.kill() + process.join() + + receive_connection.close() + + try: + send_connection.close() + except OSError: + pass diff --git a/app/services/image_validation.py b/app/services/image_validation.py new file mode 100644 index 0000000..49a8da4 --- /dev/null +++ b/app/services/image_validation.py @@ -0,0 +1,41 @@ +from io import BytesIO +from PIL import Image, UnidentifiedImageError +from fastapi import HTTPException + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +ALLOWED = {"PNG": "image/png", "JPEG": "image/jpeg"} + + +def validate_image_file(fileobj, declared_type: str): + """Validate a seekable, size-limited file without loading its full encoded bytes.""" + try: + fileobj.seek(0) + with Image.open(fileobj) as image: + fmt = image.format + width, height = image.size + if fmt not in ALLOWED: + raise HTTPException(415, "Only PNG and JPEG are supported") + if width * height > MAX_PIXELS: + raise HTTPException(413, "Image exceeds 40 million pixels") + image.verify() + fileobj.seek(0) + with Image.open(fileobj) as image: + image.load() + except HTTPException: + raise + except (UnidentifiedImageError, OSError, ValueError, SyntaxError, Image.DecompressionBombError): + raise HTTPException(415, "Invalid or corrupted image") from None + actual_type = ALLOWED[fmt] + if declared_type != actual_type: + raise HTTPException(415, "Content-Type does not match image content") + fileobj.seek(0) + return actual_type, width, height + + +def validate_image(data: bytes, declared_type: str): + if not data: + raise HTTPException(415, "Empty image") + if len(data) > MAX_BYTES: + raise HTTPException(413, "Image exceeds 10 MiB") + return validate_image_file(BytesIO(data), declared_type) diff --git a/app/services/message_service.py b/app/services/message_service.py new file mode 100644 index 0000000..ba0e2d4 --- /dev/null +++ b/app/services/message_service.py @@ -0,0 +1,444 @@ +import base64 +import hashlib +import json +import time +import uuid +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +from redis import Redis +from redis.backoff import NoBackoff +from redis.exceptions import RedisError +from redis.retry import Retry +from sqlalchemy import delete, select, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session + +from app.core.config import settings +from app.db.models import Attachment, IdempotencyRecord, Message +from app.repositories.chat_repository import ChatRepository +from app.repositories.message_repository import MessageRepository + + +class MessageNotFoundError(Exception): + pass + + +class MessageAccessError(Exception): + pass + + +class InvalidMessageDataError(Exception): + pass + + +class IdempotencyConflictError(Exception): + pass + + +class MessageService: + def __init__(self, db: Session): + self.db = db + self.messages = MessageRepository(db) + self.chats = ChatRepository(db) + + self._redis_available = True + + self.redis = Redis.from_url( + settings.cache_url, + decode_responses=True, + socket_connect_timeout=0.5, + socket_timeout=0.5, + retry_on_timeout=False, + retry=Retry(NoBackoff(), 0), + ) + + @staticmethod + def encode_cursor(message: Message) -> str: + payload = { + "created_at": message.created_at.isoformat(), + "id": str(message.id), + } + + raw = json.dumps(payload).encode("utf-8") + + return ( + base64.urlsafe_b64encode(raw) + .decode("ascii") + .rstrip("=") + ) + + @staticmethod + def decode_cursor(cursor: str | None): + if not cursor: + return None, None + + try: + padded = cursor + "=" * (-len(cursor) % 4) + raw = base64.urlsafe_b64decode(padded.encode("ascii")) + payload = json.loads(raw.decode("utf-8")) + + created_at = datetime.fromisoformat(payload["created_at"]) + message_id = uuid.UUID(payload["id"]) + + if created_at.tzinfo is None: + raise ValueError("Cursor timestamp must include timezone") + + return created_at, message_id + + except (ValueError, TypeError, KeyError, UnicodeError): + raise InvalidMessageDataError("Invalid cursor") from None + + def _version_key(self, chat_id: uuid.UUID) -> str: + return f"messages:version:{chat_id}" + + def _cache_key( + self, + chat_id: uuid.UUID, + version: int, + limit: int, + cursor: str | None, + ) -> str: + cursor_hash = hashlib.sha256( + (cursor or "first-page").encode("utf-8") + ).hexdigest()[:16] + + return ( + f"messages:{chat_id}:v{version}:" + f"limit:{limit}:cursor:{cursor_hash}" + ) + + def _get_version(self, chat_id: uuid.UUID) -> int: + if not self._redis_available: + return 0 + + try: + value = self.redis.get(self._version_key(chat_id)) + return int(value or 0) + except (RedisError, ValueError): + self._redis_available = False + return 0 + + def _bump_version(self, chat_id: uuid.UUID) -> None: + if not self._redis_available: + return + + try: + self.redis.incr(self._version_key(chat_id)) + except RedisError: + self._redis_available = False + + @staticmethod + def _serialize(messages, next_cursor): + return json.dumps({ + "items": [ + { + "id": str(message.id), + "chat_id": str(message.chat_id), + "sender_id": str(message.sender_id), + "text": message.text, + "attachment_ids": message.attachment_ids or [], + "created_at": message.created_at.isoformat(), + } + for message in messages + ], + "next_cursor": next_cursor, + }) + + @staticmethod + def _deserialize(payload: str): + data = json.loads(payload) + + messages = [ + SimpleNamespace( + id=uuid.UUID(item["id"]), + chat_id=uuid.UUID(item["chat_id"]), + sender_id=uuid.UUID(item["sender_id"]), + text=item["text"], + attachment_ids=item.get("attachment_ids", []), + created_at=datetime.fromisoformat(item["created_at"]), + ) + for item in data["items"] + ] + + return messages, data["next_cursor"] + + def _read_cache(self, key: str): + if not self._redis_available: + return None + + try: + payload = self.redis.get(key) + + if payload is None: + return None + + return self._deserialize(payload) + + except RedisError: + self._redis_available = False + return None + except (ValueError, TypeError, KeyError): + return None + + def _load_from_database( + self, + chat_id, + limit, + cursor_created_at, + cursor_id, + ): + rows = self.messages.get_history( + chat_id=chat_id, + limit=limit + 1, + cursor_created_at=cursor_created_at, + cursor_id=cursor_id, + ) + + has_more = len(rows) > limit + items = rows[:limit] + + next_cursor = ( + self.encode_cursor(items[-1]) + if has_more and items + else None + ) + + return items, next_cursor + + def create_message( + self, + chat_id: uuid.UUID, + sender_id: uuid.UUID, + text: str, + idempotency_key: str, + attachment_ids: list[str] | None = None, + ) -> Message: + text = text.strip() + attachment_ids = attachment_ids or [] + if (not text and not attachment_ids) or len(text) > 4000 or len(attachment_ids) > 10: + raise InvalidMessageDataError("Invalid message text") + + # Access is checked for every retry, before consulting the key. + if not self.chats.is_member(chat_id, sender_id): + raise MessageNotFoundError() + + if len(attachment_ids) != len(set(attachment_ids)): + raise InvalidMessageDataError("Duplicate attachment IDs") + if attachment_ids: + try: + parsed_attachment_ids = [uuid.UUID(value) for value in attachment_ids] + except (ValueError, TypeError): + raise InvalidMessageDataError("Invalid attachment UUID") from None + + attachments = list( + self.db.scalars( + select(Attachment).where( + Attachment.id.in_(parsed_attachment_ids) + ) + ).all() + ) + + if ( + len(attachments) != len(parsed_attachment_ids) + or any(attachment.chat_id != chat_id for attachment in attachments) + ): + raise MessageNotFoundError() + + method = "POST" + path = f"/api/v1/chats/{chat_id}/messages" + request_hash = hashlib.sha256( + json.dumps({"text": text, "attachment_ids": attachment_ids}, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + now = datetime.now(timezone.utc) + expires_at = now + timedelta(seconds=max(settings.idempotency_ttl_seconds, 86400)) + scope = ( + IdempotencyRecord.user_id == sender_id, + IdempotencyRecord.method == method, + IdempotencyRecord.path == path, + IdempotencyRecord.idempotency_key == idempotency_key, + ) + + try: + # An expired key may be reused. Deletion and claim are serialized by + # PostgreSQL's unique constraint for the scope. + self.db.execute( + delete(IdempotencyRecord).where(*scope, IdempotencyRecord.expires_at <= now) + ) + claim = ( + pg_insert(IdempotencyRecord) + .values( + id=uuid.uuid4(), user_id=sender_id, method=method, path=path, + idempotency_key=idempotency_key, request_hash=request_hash, + message_id=None, created_at=now, expires_at=expires_at, + ) + .on_conflict_do_nothing(constraint="uq_idempotency_scope_key") + .returning(IdempotencyRecord.id) + ) + claimed_id = self.db.execute(claim).scalar_one_or_none() + + if claimed_id is None: + existing = self.db.execute( + select(IdempotencyRecord).where(*scope).with_for_update() + ).scalar_one_or_none() + if existing is None: + # Rare expiration race: retry once in a fresh transaction. + self.db.rollback() + return self.create_message(chat_id, sender_id, text, idempotency_key, attachment_ids) + if existing.request_hash != request_hash: + self.db.rollback() + raise IdempotencyConflictError() + if existing.message_id is None: + raise RuntimeError("Committed idempotency record has no message") + message = self.db.get(Message, existing.message_id) + if message is None: + raise RuntimeError("Idempotency record references a missing message") + self.db.commit() + return message + + message = self.messages.create( + chat_id=chat_id, sender_id=sender_id, text=text, attachment_ids=attachment_ids + ) + self.db.execute( + update(IdempotencyRecord) + .where(IdempotencyRecord.id == claimed_id) + .values(message_id=message.id) + ) + self.db.commit() + self.db.refresh(message) + self._bump_version(chat_id) + return message + except IdempotencyConflictError: + raise + except Exception: + self.db.rollback() + raise + + def get_history( + self, + chat_id: uuid.UUID, + user_id: uuid.UUID, + limit: int = 50, + cursor: str | None = None, + ): + if not 1 <= limit <= 100: + raise InvalidMessageDataError( + "Limit must be between 1 and 100" + ) + + if not self.chats.is_member(chat_id, user_id): + raise MessageNotFoundError() + + cursor_created_at, cursor_id = self.decode_cursor(cursor) + + version = self._get_version(chat_id) + + cache_key = self._cache_key( + chat_id, + version, + limit, + cursor, + ) + + cached = self._read_cache(cache_key) + + if cached is not None: + return cached + + if not self._redis_available: + return self._load_from_database( + chat_id, + limit, + cursor_created_at, + cursor_id, + ) + + lock_key = f"lock:{cache_key}" + lock_token = str(uuid.uuid4()) + + try: + acquired = self.redis.set( + lock_key, + lock_token, + nx=True, + ex=10, + ) + except RedisError: + self._redis_available = False + return self._load_from_database( + chat_id, + limit, + cursor_created_at, + cursor_id, + ) + + if not acquired: + for _ in range(10): + time.sleep(0.05) + + cached = self._read_cache(cache_key) + + if cached is not None: + return cached + + if not self._redis_available: + break + + return self._load_from_database( + chat_id, + limit, + cursor_created_at, + cursor_id, + ) + + try: + cached = self._read_cache(cache_key) + + if cached is not None: + return cached + + if not self._redis_available: + return self._load_from_database( + chat_id, + limit, + cursor_created_at, + cursor_id, + ) + + items, next_cursor = self._load_from_database( + chat_id, + limit, + cursor_created_at, + cursor_id, + ) + + if self._redis_available and self._get_version(chat_id) == version: + try: + self.redis.setex( + cache_key, + settings.message_cache_ttl_seconds, + self._serialize(items, next_cursor), + ) + except RedisError: + self._redis_available = False + + return items, next_cursor + + finally: + if self._redis_available: + try: + self.redis.eval( + """ + if redis.call('get', KEYS[1]) == ARGV[1] + then + return redis.call('del', KEYS[1]) + else + return 0 + end + """, + 1, + lock_key, + lock_token, + ) + except RedisError: + self._redis_available = False diff --git a/app/services/object_storage.py b/app/services/object_storage.py new file mode 100644 index 0000000..bedc3c4 --- /dev/null +++ b/app/services/object_storage.py @@ -0,0 +1,41 @@ +import os +import boto3 +from botocore.config import Config +from app.core.config import settings + + +def client(endpoint_url=None): + return boto3.client( + "s3", endpoint_url=endpoint_url or settings.s3_endpoint_url, + aws_access_key_id=settings.s3_access_key, + aws_secret_access_key=settings.s3_secret_key, + region_name=settings.s3_region, + verify=os.getenv("TLS_CA_FILE") if (endpoint_url or settings.s3_endpoint_url).startswith("https://") else True, + config=Config(signature_version="s3v4", connect_timeout=3, read_timeout=15, retries={"max_attempts": 2}, s3={"addressing_style": "path"}), + ) + + +def ensure_bucket() -> None: + s3 = client() + try: + s3.head_bucket(Bucket=settings.s3_bucket) + except Exception: + s3.create_bucket(Bucket=settings.s3_bucket) + + +def presigned_client(): + return client(endpoint_url=settings.s3_public_endpoint_url) + + +def create_presigned_upload_url(object_key: str, content_type: str) -> str: + s3 = presigned_client() + + return s3.generate_presigned_url( + ClientMethod="put_object", + Params={ + "Bucket": settings.s3_bucket, + "Key": object_key, + "ContentType": content_type, + }, + ExpiresIn=settings.presigned_upload_ttl_seconds, + ) \ No newline at end of file diff --git a/app/services/outbox_publisher.py b/app/services/outbox_publisher.py new file mode 100644 index 0000000..0014908 --- /dev/null +++ b/app/services/outbox_publisher.py @@ -0,0 +1,107 @@ +"""Publish durable PostgreSQL outbox events to RabbitMQ. + +The database row is marked published only after RabbitMQ publisher confirmation. +If the process crashes after broker confirmation but before the DB commit, the +same event is published again; consumers must therefore be idempotent. +""" + +import asyncio +import json +import logging +import random +from datetime import datetime, timezone +from typing import Any + +import aio_pika +from app.core.tls import service_ssl_context +from aio_pika import DeliveryMode, Message +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.core.config import settings +from app.db.models import OutboxEvent +from app.db.session import SessionLocal + +logger = logging.getLogger(__name__) + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +async def publish_once() -> int: + """Publish one batch; return number of confirmed events.""" + connection = await aio_pika.connect_robust( + settings.rabbitmq_url, + ssl=settings.rabbitmq_url.startswith("amqps://"), + ssl_context=service_ssl_context() if settings.rabbitmq_url.startswith("amqps://") else None, + ) + try: + channel = await connection.channel(publisher_confirms=True) + exchange = await channel.declare_exchange( + settings.rabbitmq_exchange, + aio_pika.ExchangeType.DIRECT, + durable=True, + ) + queue = await channel.declare_queue(settings.rabbitmq_queue, durable=True) + await queue.bind(exchange, routing_key=settings.rabbitmq_queue) + + published = 0 + db: Session = SessionLocal() + try: + # Lock a bounded batch. SKIP LOCKED allows multiple publisher replicas. + events = db.scalars( + select(OutboxEvent) + .where(OutboxEvent.status == "pending") + .order_by(OutboxEvent.created_at) + .with_for_update(skip_locked=True) + .limit(settings.outbox_batch_size) + ).all() + + for event in events: + body = json.dumps(event.payload, separators=(",", ":")).encode("utf-8") + message = Message( + body=body, + content_type="application/json", + delivery_mode=DeliveryMode.PERSISTENT, + message_id=str(event.id), + type=event.event_type, + timestamp=utc_now(), + ) + await exchange.publish(message, routing_key=settings.rabbitmq_queue, mandatory=True) + event.status = "published" + event.published_at = utc_now() + event.attempts += 1 + published += 1 + + db.commit() + return published + except Exception: + db.rollback() + raise + finally: + db.close() + finally: + await connection.close() + + +async def run_publisher() -> None: + delay = settings.outbox_poll_interval_seconds + while True: + try: + count = await publish_once() + if count: + logger.info("Published %s outbox events", count) + delay = settings.outbox_poll_interval_seconds + except asyncio.CancelledError: + raise + except Exception: + logger.exception("Outbox publisher iteration failed") + delay = min(max(delay * 2, settings.outbox_poll_interval_seconds), 15.0) + delay += random.uniform(0, delay * 0.2) + await asyncio.sleep(delay) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + asyncio.run(run_publisher()) diff --git a/app/services/user_service.py b/app/services/user_service.py new file mode 100644 index 0000000..65eeb12 --- /dev/null +++ b/app/services/user_service.py @@ -0,0 +1,89 @@ +import os +import re + +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from pwdlib import PasswordHash +from pwdlib.hashers.argon2 import Argon2Hasher + +from app.db.models import User +from app.repositories.user_repository import UserRepository + + +password_hasher = PasswordHash(( + Argon2Hasher( + time_cost=int(os.getenv("ARGON2_ITERATIONS", "2")), + memory_cost=int(os.getenv("ARGON2_MEMORY_KIB", "19456")), + parallelism=int(os.getenv("ARGON2_PARALLELISM", "1")), + ), +)) + + +class UsernameAlreadyExistsError(Exception): + pass + + +class UserService: + + def __init__(self, db: Session): + self.db = db + self.users = UserRepository(db) + + def register( + self, + username: str, + display_name: str, + password: str, + ) -> User: + + if not re.fullmatch(r"[a-z0-9_]{3,32}", username): + raise ValueError("Invalid username") + + if not display_name or not display_name.strip(): + raise ValueError("Display name cannot be empty") + + if len(display_name) > 100: + raise ValueError("Display name is too long") + + if not 12 <= len(password) <= 128: + raise ValueError("Invalid password length") + + if self.users.get_by_username(username) is not None: + raise UsernameAlreadyExistsError( + "Username already exists" + ) + + password_hash = password_hasher.hash(password) + + try: + user = self.users.create( + username=username, + display_name=display_name, + password_hash=password_hash, + ) + + self.db.commit() + self.db.refresh(user) + + return user + + except IntegrityError as exc: + self.db.rollback() + + raise UsernameAlreadyExistsError( + "Username already exists" + ) from exc + + except Exception: + self.db.rollback() + raise + + def verify_password( + self, + user: User, + password: str, + ) -> bool: + return password_hasher.verify( + password, + user.password_hash, + ) \ No newline at end of file diff --git a/app/store.py b/app/store.py new file mode 100644 index 0000000..4802821 --- /dev/null +++ b/app/store.py @@ -0,0 +1,193 @@ +import base64 +import bisect +import binascii +import json +import os +import threading +import uuid +from datetime import datetime, timezone +from typing import Dict, List, Optional, Tuple + +from .models import Chat, Message, User + + +Key = Tuple[datetime, str] + + +def _encode_cursor(kind: str, scope: str, item) -> str: + payload = { + "v": 1, + "kind": kind, + "scope": scope, + "created_at": item.created_at.isoformat(), + "id": item.id, + } + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def _decode_cursor(cursor: str, kind: str, scope: str) -> Key: + try: + if not cursor or len(cursor) > 4096: + raise ValueError + padded = cursor + "=" * (-len(cursor) % 4) + raw = base64.b64decode(padded, altchars=b"-_", validate=True) + payload = json.loads(raw.decode("utf-8")) + if not isinstance(payload, dict) or payload.get("v") != 1: + raise ValueError + if payload.get("kind") != kind or payload.get("scope") != scope: + raise ValueError + item_id = payload.get("id") + timestamp = payload.get("created_at") + if not isinstance(item_id, str) or not isinstance(timestamp, str): + raise ValueError + created_at = datetime.fromisoformat(timestamp) + if created_at.tzinfo is None: + raise ValueError + created_at = created_at.astimezone(timezone.utc) + return created_at, item_id + except ( + ValueError, + TypeError, + KeyError, + UnicodeDecodeError, + json.JSONDecodeError, + binascii.Error, + ): + raise ValueError("invalid_request") from None + + +class InMemoryStore: + def __init__(self): + self._lock = threading.RLock() + self.users: Dict[str, User] = {} + self.chats: Dict[str, Chat] = {} + self.messages: Dict[str, List[Message]] = {} + self.usernames: Dict[str, str] = {} + self.message_keys: Dict[str, List[Key]] = {} + self.message_index_enabled = os.getenv("MESSAGE_INDEX_ENABLED", "1") != "0" + + def create_user(self, username: str, display_name: str) -> User: + with self._lock: + if username in self.usernames: + raise ValueError("conflict") + user_id = str(uuid.uuid4()) + user = User( + id=user_id, + username=username, + display_name=display_name, + created_at=datetime.now(timezone.utc), + ) + self.users[user_id] = user + self.usernames[username] = user_id + return user + + def get_user(self, user_id: str) -> Optional[User]: + with self._lock: + return self.users.get(user_id) + + def create_chat(self, title: str, member_ids: List[str], creator_id: str) -> Chat: + with self._lock: + if creator_id not in member_ids or len(member_ids) != len(set(member_ids)): + raise ValueError("invalid_request") + if any(member_id not in self.users for member_id in member_ids): + raise ValueError("invalid_request") + + chat_id = str(uuid.uuid4()) + chat = Chat( + id=chat_id, + title=title, + member_ids=list(member_ids), + created_at=datetime.now(timezone.utc), + ) + self.chats[chat_id] = chat + self.messages[chat_id] = [] + self.message_keys[chat_id] = [] + return chat + + def get_chat(self, chat_id: str, user_id: str) -> Optional[Chat]: + with self._lock: + chat = self.chats.get(chat_id) + if chat is None or user_id not in chat.member_ids: + return None + return chat + + def list_chats( + self, user_id: str, limit: int, cursor: Optional[str] + ) -> Tuple[List[Chat], Optional[str]]: + with self._lock: + user_chats = [chat for chat in self.chats.values() if user_id in chat.member_ids] + user_chats.sort(key=lambda chat: (chat.created_at, chat.id)) + keys = [(chat.created_at, chat.id) for chat in user_chats] + + start_index = 0 + if cursor: + target_key = _decode_cursor(cursor, kind="chats", scope=user_id) + start_index = bisect.bisect_right(keys, target_key) + + page_with_extra = user_chats[start_index : start_index + limit + 1] + has_next = len(page_with_extra) > limit + page = page_with_extra[:limit] + next_cursor = None + if has_next and page: + next_cursor = _encode_cursor("chats", user_id, page[-1]) + return page, next_cursor + + def create_message(self, chat_id: str, sender_id: str, text: str) -> Message: + with self._lock: + chat = self.chats.get(chat_id) + if chat is None or sender_id not in chat.member_ids: + raise ValueError("not_found") + + message = Message( + id=str(uuid.uuid4()), + chat_id=chat_id, + sender_id=sender_id, + text=text, + created_at=datetime.now(timezone.utc), + ) + messages = self.messages[chat_id] + keys = self.message_keys[chat_id] + key = (message.created_at, message.id) + + if self.message_index_enabled: + index = bisect.bisect_right(keys, key) + keys.insert(index, key) + messages.insert(index, message) + else: + current_keys = [(item.created_at, item.id) for item in messages] + index = bisect.bisect_right(current_keys, key) + messages.insert(index, message) + keys.insert(index, key) + return message + + def list_messages( + self, + chat_id: str, + user_id: str, + limit: int, + cursor: Optional[str], + ) -> Tuple[List[Message], Optional[str]]: + with self._lock: + chat = self.chats.get(chat_id) + if chat is None or user_id not in chat.member_ids: + raise ValueError("not_found") + + messages = self.messages[chat_id] + if self.message_index_enabled: + keys = self.message_keys[chat_id] + else: + keys = [(item.created_at, item.id) for item in messages] + + start_index = 0 + if cursor: + target_key = _decode_cursor(cursor, kind="messages", scope=chat_id) + start_index = bisect.bisect_right(keys, target_key) + + page_with_extra = messages[start_index : start_index + limit + 1] + has_next = len(page_with_extra) > limit + page = page_with_extra[:limit] + next_cursor = None + if has_next and page: + next_cursor = _encode_cursor("messages", chat_id, page[-1]) + return page, next_cursor diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 0000000..670bb34 --- /dev/null +++ b/app/worker.py @@ -0,0 +1,998 @@ + +"""RabbitMQ consumer process for durable image processing.""" + +import asyncio +import json +import logging +import random +import socket +import threading +import uuid +import time + +from datetime import datetime, timedelta, timezone + +import aio_pika +from aio_pika import DeliveryMode, Message +from botocore.exceptions import BotoCoreError, ClientError +from sqlalchemy import select, update +from sqlalchemy.exc import SQLAlchemyError + +from app.core.config import settings +from app.core.tls import service_ssl_context +from app.db.models import Attachment, AttachmentVariant, ProcessingJob +from app.db.session import SessionLocal +from app.services.image_processor import ( + ImageProcessingTimeout, + create_variants_with_timeout, +) +from app.services.object_storage import client as s3_client + +from app.core.worker_telemetry import ( + process_job_with_telemetry, + setup_worker_telemetry, +) + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", +) + +logger = logging.getLogger("attachment-worker") + +WORKER_ID = ( + f"{socket.gethostname()[:40]}-{uuid.uuid4().hex[:16]}" +) + +DEAD_ROUTING_KEY = f"{settings.rabbitmq_queue}.dead" + +PROCESSING_SEMAPHORE = None + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _safe_uuid(value) -> uuid.UUID: + return uuid.UUID(str(value)) + + +def _claim_job(job_id, attachment_id, pipeline_version) -> dict: + """Atomically claim a job or return the reason it cannot be claimed.""" + + job_id = _safe_uuid(job_id) + attachment_id = _safe_uuid(attachment_id) + pipeline_version = int(pipeline_version) + + db = SessionLocal() + + try: + job = db.scalar( + select(ProcessingJob) + .where( + ProcessingJob.id == job_id, + ProcessingJob.attachment_id == attachment_id, + ProcessingJob.pipeline_version == pipeline_version, + ) + .with_for_update() + ) + + if job is None: + db.rollback() + return {"state": "missing"} + + attachment = db.get(Attachment, attachment_id) + + if attachment is None: + db.rollback() + return {"state": "missing"} + + now = utc_now() + + if attachment.status == "ready": + db.commit() + return {"state": "ready"} + + if job.status == "failed": + db.commit() + return {"state": "failed"} + + if ( + job.status == "processing" + and job.lease_until is not None + and job.lease_until > now + ): + delay = (job.lease_until - now).total_seconds() + 1 + + db.commit() + + return { + "state": "busy", + "delay": max(1.0, delay), + } + + if job.available_at > now: + delay = (job.available_at - now).total_seconds() + 0.5 + + db.commit() + + return { + "state": "deferred", + "delay": max(1.0, delay), + } + + if job.attempt >= settings.worker_max_attempts: + job.status = "failed" + job.worker_id = None + job.lease_until = None + job.last_error_code = "processing_attempts_exhausted" + + attachment.status = "failed" + attachment.error_code = "processing_attempts_exhausted" + + db.commit() + + return {"state": "failed"} + + claim_worker_id = f"{WORKER_ID}:{uuid.uuid4().hex[:16]}" + + job.status = "processing" + job.worker_id = claim_worker_id + job.lease_until = now + timedelta( + seconds=settings.worker_lease_seconds + ) + job.attempt += 1 + + attachment.status = "processing" + attachment.error_code = None + + claimed_attempt = job.attempt + + queue_lag_ms = max( + 0.0, + (now - job.available_at).total_seconds() * 1000, + ) + + db.commit() + + logger.info( + "Claimed job=%s attachment=%s attempt=%s queue_lag_ms=%.2f", + job_id, + attachment_id, + claimed_attempt, + queue_lag_ms, + ) + + return { + "state": "claimed", + "worker_id": claim_worker_id, + "attempt": claimed_attempt, + "queue_lag_ms": queue_lag_ms, + } + + except Exception: + db.rollback() + raise + + finally: + db.close() + + +def _renew_lease( + job_id: uuid.UUID, + worker_id: str, + attempt: int, + stop_event: threading.Event, + lost_event: threading.Event, +) -> None: + """Renew the lease while image processing is running.""" + + interval = max( + 5, + settings.worker_lease_seconds // 3, + ) + + while not stop_event.wait(interval): + db = SessionLocal() + + try: + result = db.execute( + update(ProcessingJob) + .where( + ProcessingJob.id == job_id, + ProcessingJob.worker_id == worker_id, + ProcessingJob.attempt == attempt, + ProcessingJob.status == "processing", + ) + .values( + lease_until=utc_now() + + timedelta(seconds=settings.worker_lease_seconds) + ) + ) + + db.commit() + + if result.rowcount != 1: + lost_event.set() + + logger.warning( + "Lease ownership lost: job=%s worker=%s", + job_id, + worker_id, + ) + + return + + except Exception: + db.rollback() + lost_event.set() + + logger.exception( + "Could not renew lease for job=%s", + job_id, + ) + + return + + finally: + db.close() + + +def _fail_job( + job_id, + attachment_id, + worker_id, + attempt, + error_code, + permanent=False, +) -> dict: + """Record a failure only if this worker still owns the job.""" + + db = SessionLocal() + + try: + job = db.scalar( + select(ProcessingJob) + .where(ProcessingJob.id == job_id) + .with_for_update() + ) + + attachment = db.scalar( + select(Attachment) + .where(Attachment.id == attachment_id) + .with_for_update() + ) + + if job is None or attachment is None: + db.rollback() + return {"state": "missing"} + + if ( + job.status != "processing" + or job.worker_id != worker_id + or job.attempt != attempt + ): + db.rollback() + + logger.warning( + "Ignoring failure from stale worker: job=%s worker=%s", + job_id, + worker_id, + ) + + return { + "state": "stale", + "delay": float(settings.worker_lease_seconds), + } + + exhausted = job.attempt >= settings.worker_max_attempts + + job.last_error_code = error_code + + if permanent or exhausted: + job.status = "failed" + job.worker_id = None + job.lease_until = None + + attachment.status = "failed" + attachment.error_code = error_code + + db.commit() + + logger.error( + "Job permanently failed: job=%s error=%s", + job_id, + error_code, + ) + + return {"state": "failed"} + + delay = min( + settings.worker_retry_base_seconds + * (2 ** max(job.attempt - 1, 0)), + settings.worker_retry_max_seconds, + ) + + delay += random.uniform(0, delay * 0.25) + + job.status = "queued" + job.available_at = utc_now() + timedelta(seconds=delay) + job.lease_until = None + job.worker_id = None + + attachment.status = "queued" + attachment.error_code = error_code + + db.commit() + + logger.warning( + "Job scheduled for retry: job=%s delay=%.2fs error=%s", + job_id, + delay, + error_code, + ) + + return { + "state": "retry", + "delay": delay, + } + + except Exception: + db.rollback() + raise + + finally: + db.close() + + +def _process_impl(payload: dict) -> dict: + """Process one image using a fenced, renewable job lease.""" + + job_id = _safe_uuid(payload["job_id"]) + attachment_id = _safe_uuid(payload["attachment_id"]) + pipeline_version = int(payload["pipeline_version"]) + + claim = _claim_job( + job_id, + attachment_id, + pipeline_version, + ) + + if claim["state"] != "claimed": + return claim + + worker_id = claim["worker_id"] + attempt = claim["attempt"] + queue_lag_ms = claim["queue_lag_ms"] + processing_started_at = time.perf_counter() + + stop_event = threading.Event() + lost_event = threading.Event() + + heartbeat = threading.Thread( + target=_renew_lease, + args=( + job_id, + worker_id, + attempt, + stop_event, + lost_event, + ), + daemon=True, + ) + + heartbeat.start() + + staging_key = None + db = None + + try: + db = SessionLocal() + + attachment = db.get(Attachment, attachment_id) + + if attachment is None: + raise ValueError("attachment_missing") + + staging_key = attachment.staging_key + object_key = attachment.object_key + content_type = attachment.content_type + + if not staging_key: + raise ValueError("staging_missing") + + db.close() + db = None + + if lost_event.is_set(): + return { + "state": "retry", + "delay": float(settings.worker_lease_seconds), + } + + s3 = s3_client() + + response = s3.get_object( + Bucket=settings.s3_bucket, + Key=staging_key, + ) + + body = response["Body"] + + try: + attachment_data = body.read( + settings.max_upload_bytes + 1 + ) + finally: + body.close() + + if len(attachment_data) > settings.max_upload_bytes: + raise ValueError("image_too_large") + + if lost_event.is_set(): + return { + "state": "retry", + "delay": float(settings.worker_lease_seconds), + } + + variants = create_variants_with_timeout( + attachment_data, + settings.image_processing_timeout_seconds, + ) + + # The original object uses a deterministic key. + s3.put_object( + Bucket=settings.s3_bucket, + Key=object_key, + Body=attachment_data, + ContentType=content_type, + ) + + variant_records = [] + + for variant in variants: + key = ( + f"variants/{attachment_id}/" + f"v{pipeline_version}/" + f"{variant.name}.{variant.extension}" + ) + + s3.put_object( + Bucket=settings.s3_bucket, + Key=key, + Body=variant.data, + ContentType=variant.content_type, + ) + + variant_records.append( + { + "name": variant.name, + "key": key, + "content_type": variant.content_type, + "width": variant.width, + "height": variant.height, + "size_bytes": len(variant.data), + "sha256": variant.sha256, + } + ) + + if lost_event.is_set(): + logger.warning( + "Worker lost lease before final commit: job=%s", + job_id, + ) + + return { + "state": "retry", + "delay": float(settings.worker_lease_seconds), + } + + # Use a fresh transaction and verify the fencing token. + db = SessionLocal() + + job = db.scalar( + select(ProcessingJob) + .where(ProcessingJob.id == job_id) + .with_for_update() + ) + + attachment = db.scalar( + select(Attachment) + .where(Attachment.id == attachment_id) + .with_for_update() + ) + + if job is None or attachment is None: + db.rollback() + return {"state": "missing"} + + if ( + job.status != "processing" + or job.worker_id != worker_id + or job.attempt != attempt + or job.lease_until is None + or job.lease_until <= utc_now() + or lost_event.is_set() + ): + db.rollback() + + logger.warning( + "Fencing check rejected stale worker: job=%s", + job_id, + ) + + return { + "state": "retry", + "delay": float(settings.worker_lease_seconds), + } + + for item in variant_records: + existing = db.scalar( + select(AttachmentVariant) + .where( + AttachmentVariant.attachment_id == attachment_id, + AttachmentVariant.variant == item["name"], + ) + ) + + if existing is None: + existing = AttachmentVariant( + attachment_id=attachment_id, + variant=item["name"], + object_key=item["key"], + content_type=item["content_type"], + width=item["width"], + height=item["height"], + size_bytes=item["size_bytes"], + sha256=item["sha256"], + ) + + db.add(existing) + + else: + existing.object_key = item["key"] + existing.content_type = item["content_type"] + existing.width = item["width"] + existing.height = item["height"] + existing.size_bytes = item["size_bytes"] + existing.sha256 = item["sha256"] + + job.status = "ready" + job.lease_until = None + job.worker_id = None + job.last_error_code = None + + attachment.status = "ready" + attachment.error_code = None + attachment.updated_at = utc_now() + + db.commit() + + processing_elapsed_ms = ( + time.perf_counter() - processing_started_at + ) * 1000 + + logger.info( + ( + "Image processing completed: attachment=%s " + "attempt=%s queue_lag_ms=%.2f processing_elapsed_ms=%.2f" + ), + attachment_id, + attempt, + queue_lag_ms, + processing_elapsed_ms, + ) + + db.close() + db = None + + # Staging cleanup is performed only after the final DB commit. + try: + s3.delete_object( + Bucket=settings.s3_bucket, + Key=staging_key, + ) + + cleanup_db = SessionLocal() + + try: + cleanup_attachment = cleanup_db.scalar( + select(Attachment) + .where(Attachment.id == attachment_id) + .with_for_update() + ) + + if ( + cleanup_attachment is not None + and cleanup_attachment.status == "ready" + and cleanup_attachment.staging_key == staging_key + ): + cleanup_attachment.staging_key = None + cleanup_db.commit() + else: + cleanup_db.rollback() + + finally: + cleanup_db.close() + + except Exception: + logger.warning( + "Could not delete staging object for attachment=%s", + attachment_id, + exc_info=True, + ) + + return {"state": "ready"} + + except ImageProcessingTimeout: + if db is not None: + db.rollback() + + stop_event.set() + heartbeat.join(timeout=2) + + logger.warning( + "Image processing timed out: job=%s attachment=%s timeout=%ss", + job_id, + attachment_id, + settings.image_processing_timeout_seconds, + ) + + return _fail_job( + job_id, + attachment_id, + worker_id, + attempt, + "image_processing_timeout", + permanent=False, + ) + + except ValueError as exc: + if db is not None: + db.rollback() + + stop_event.set() + heartbeat.join(timeout=2) + + code = str(exc)[:64] or "invalid_image" + + return _fail_job( + job_id, + attachment_id, + worker_id, + attempt, + code, + permanent=True, + ) + + except ClientError as exc: + if db is not None: + db.rollback() + + stop_event.set() + heartbeat.join(timeout=2) + + error = exc.response.get("Error", {}) + error_code = error.get("Code", "") + + if error_code in { + "NoSuchKey", + "NoSuchBucket", + "404", + "NotFound", + }: + return _fail_job( + job_id, + attachment_id, + worker_id, + attempt, + "staging_missing", + permanent=True, + ) + + return _fail_job( + job_id, + attachment_id, + worker_id, + attempt, + "storage_unavailable", + permanent=False, + ) + + except (BotoCoreError, SQLAlchemyError): + if db is not None: + db.rollback() + + stop_event.set() + heartbeat.join(timeout=2) + + return _fail_job( + job_id, + attachment_id, + worker_id, + attempt, + "processing_unavailable", + permanent=False, + ) + + except Exception: + if db is not None: + db.rollback() + + logger.exception( + "Unexpected processing error: job=%s", + job_id, + ) + + stop_event.set() + heartbeat.join(timeout=2) + + return _fail_job( + job_id, + attachment_id, + worker_id, + attempt, + "processing_failed", + permanent=False, + ) + + finally: + stop_event.set() + heartbeat.join(timeout=2) + + if db is not None: + db.close() + +def _process(payload: dict) -> dict: + return process_job_with_telemetry( + _process_impl, + payload, + ) + + +async def _publish_delayed( + exchange, + payload: dict, + delay_seconds: float, +) -> None: + """Publish a persistent message to the TTL retry queue.""" + + delay_ms = max( + 1000, + int(delay_seconds * 1000), + ) + + message = Message( + body=json.dumps( + payload, + separators=(",", ":"), + ).encode("utf-8"), + content_type="application/json", + delivery_mode=DeliveryMode.PERSISTENT, + message_id=str(uuid.uuid4()), + type="attachment.process.retry", + expiration=timedelta(milliseconds=delay_ms), + ) + + await exchange.publish( + message, + routing_key=f"{settings.rabbitmq_queue}.retry", + mandatory=True, + ) + + logger.info( + "Scheduled delayed delivery in %.2fs", + delay_ms / 1000, + ) + + +async def _quarantine_message(exchange, message) -> None: + """Move an invalid message to the durable dead-letter queue.""" + + dead_message = Message( + body=message.body, + content_type=message.content_type or "application/octet-stream", + delivery_mode=DeliveryMode.PERSISTENT, + message_id=message.message_id or str(uuid.uuid4()), + type="attachment.process.invalid", + ) + + await exchange.publish( + dead_message, + routing_key=DEAD_ROUTING_KEY, + mandatory=True, + ) + + await message.ack() + + logger.error( + "Invalid message moved to dead-letter queue: message_id=%s", + message.message_id, + ) + + +async def handle_message( + message: aio_pika.IncomingMessage, + retry_exchange, + dead_exchange, + semaphore: asyncio.Semaphore, +) -> None: + """Consume, process, retry or quarantine one RabbitMQ message.""" + + async with semaphore: + try: + payload = json.loads(message.body.decode("utf-8")) + + if not isinstance(payload, dict): + raise ValueError("invalid_payload") + + for field in ( + "job_id", + "attachment_id", + "pipeline_version", + ): + if field not in payload: + raise ValueError("missing_payload_field") + + _safe_uuid(payload["job_id"]) + _safe_uuid(payload["attachment_id"]) + int(payload["pipeline_version"]) + + except (ValueError, TypeError, KeyError, UnicodeDecodeError): + try: + await _quarantine_message( + dead_exchange, + message, + ) + except Exception: + logger.exception( + "Could not quarantine invalid message" + ) + + await message.nack(requeue=True) + + return + + try: + result = await asyncio.to_thread( + _process, + payload, + ) + + state = result["state"] + + if state in { + "busy", + "deferred", + "retry", + }: + await _publish_delayed( + retry_exchange, + payload, + result.get("delay", 1.0), + ) + + await message.ack() + + logger.info( + "Message handled: job=%s state=%s", + payload["job_id"], + state, + ) + + except asyncio.CancelledError: + raise + + except Exception: + logger.exception( + "Message processing failed: message_id=%s", + message.message_id, + ) + + # If delayed publication or DB processing fails, + # preserve the original message. + await message.nack(requeue=True) + + +async def main() -> None: + """Start the durable RabbitMQ consumer.""" + + setup_worker_telemetry() + + connection = await aio_pika.connect_robust( + settings.rabbitmq_url, + ssl=settings.rabbitmq_url.startswith("amqps://"), + ssl_context=service_ssl_context() if settings.rabbitmq_url.startswith("amqps://") else None, + ) + + async with connection: + channel = await connection.channel( + publisher_confirms=True + ) + + await channel.set_qos( + prefetch_count=settings.rabbitmq_prefetch + ) + + exchange = await channel.declare_exchange( + settings.rabbitmq_exchange, + aio_pika.ExchangeType.DIRECT, + durable=True, + ) + + queue = await channel.declare_queue( + settings.rabbitmq_queue, + durable=True, + ) + + await queue.bind( + exchange, + routing_key=settings.rabbitmq_queue, + ) + + # Delayed retry exchange and queue. + retry_exchange = await channel.declare_exchange( + f"{settings.rabbitmq_exchange}.retry", + aio_pika.ExchangeType.DIRECT, + durable=True, + ) + + retry_queue = await channel.declare_queue( + f"{settings.rabbitmq_queue}.retry", + durable=True, + arguments={ + "x-dead-letter-exchange": settings.rabbitmq_exchange, + "x-dead-letter-routing-key": settings.rabbitmq_queue, + }, + ) + + await retry_queue.bind( + retry_exchange, + routing_key=f"{settings.rabbitmq_queue}.retry", + ) + + # Durable quarantine queue for invalid messages. + dead_exchange = await channel.declare_exchange( + f"{settings.rabbitmq_exchange}.dead", + aio_pika.ExchangeType.DIRECT, + durable=True, + ) + + dead_queue = await channel.declare_queue( + f"{settings.rabbitmq_queue}.dead", + durable=True, + ) + + await dead_queue.bind( + dead_exchange, + routing_key=DEAD_ROUTING_KEY, + ) + + semaphore = asyncio.Semaphore( + settings.worker_concurrency + ) + + async def consumer(message): + await handle_message( + message, + retry_exchange, + dead_exchange, + semaphore, + ) + + logger.info( + "Worker %s consuming queue=%s concurrency=%s prefetch=%s", + WORKER_ID, + settings.rabbitmq_queue, + settings.worker_concurrency, + settings.rabbitmq_prefetch, + ) + + await queue.consume(consumer) + + await asyncio.Future() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..a6ea11d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,483 @@ +services: + tls-proxy: + image: dockurr/stunnel:5.76 + volumes: + - ./infra/stunnel/stunnel.conf:/stunnel.conf:ro + - ./secrets/tls/tls-proxy.crt:/cert.crt:ro + - ./secrets/tls/tls-proxy.key:/private.key:ro + ports: + - 127.0.0.1:18443:18443 + - 127.0.0.1:19091:19090 + - 127.0.0.1:13001:13001 + restart: unless-stopped + networks: + - backend + - data_backend + security_opt: + - no-new-privileges:true + tmpfs: + - /tmp:size=16m,noexec,nosuid + healthcheck: + test: + - CMD-SHELL + - nc -z 127.0.0.1 15432 && nc -z 127.0.0.1 16379 && nc -z 127.0.0.1 15672 && + nc -z 127.0.0.1 18333 && nc -z 127.0.0.1 18443 && nc -z 127.0.0.1 14318 && + nc -z 127.0.0.1 13100 && nc -z 127.0.0.1 14320 && nc -z 127.0.0.1 13200 && + nc -z 127.0.0.1 18889 && nc -z 127.0.0.1 19090 && nc -z 127.0.0.1 13001 + interval: 5s + timeout: 3s + retries: 20 + start_period: 5s + environment: + UID: '1000' + GID: '1000' + openbao: + image: openbao/openbao:2.2.0 + command: + - server + - -config=/openbao/config.hcl + volumes: + - ./infra/openbao/config.hcl:/openbao/config.hcl:ro + - ./secrets/tls/openbao.crt:/run/tls/openbao.crt:ro + - ./secrets/tls/openbao.key:/run/tls/openbao.key:ro + - ./secrets/tls/ca.crt:/run/tls/ca.crt:ro + - openbao_data:/openbao/data + ports: + - 127.0.0.1:18200:8200 + restart: unless-stopped + networks: + - backend + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + cap_add: + - SETGID + - SETUID + postgres: + image: postgres:16.6-bookworm + environment: + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + API_DB_USER: ${API_DB_USER} + API_DB_PASSWORD: ${API_DB_PASSWORD} + WORKER_DB_USER: ${WORKER_DB_USER} + WORKER_DB_PASSWORD: ${WORKER_DB_PASSWORD} + OUTBOX_DB_USER: ${OUTBOX_DB_USER} + OUTBOX_DB_PASSWORD: ${OUTBOX_DB_PASSWORD} + volumes: + - postgres_data:/var/lib/postgresql/data + - ./postgres/init-roles.sh:/docker-entrypoint-initdb.d/10-roles.sh:ro + healthcheck: + test: + - CMD-SHELL + - pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB} + interval: 5s + timeout: 3s + retries: 15 + restart: unless-stopped + networks: + - data_backend + redis: + image: redis:7.4.2-alpine + command: + - redis-server + - --appendonly + - 'yes' + - --requirepass + - ${REDIS_ADMIN_PASSWORD} + - --aclfile + - /data/users.acl + volumes: + - redis_data:/data + healthcheck: + test: + - CMD-SHELL + - redis-cli -a "$${REDIS_ADMIN_PASSWORD}" ping | grep -q PONG + interval: 5s + timeout: 3s + retries: 10 + restart: unless-stopped + networks: + - data_backend + environment: + REDIS_ADMIN_PASSWORD: ${REDIS_ADMIN_PASSWORD} + rabbitmq: + image: rabbitmq:4.0.5-management-alpine + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD} + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: + - CMD + - rabbitmq-diagnostics + - -q + - ping + interval: 10s + timeout: 5s + retries: 10 + restart: unless-stopped + networks: + - data_backend + seaweedfs: + image: chrislusf/seaweedfs:4.46 + command: + - mini + - -dir=/data + environment: + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + S3_BUCKET: ${S3_BUCKET} + volumes: + - seaweedfs_data:/data + restart: unless-stopped + networks: + - data_backend + migrate: + build: + context: . + dockerfile: Dockerfile + volumes: + - ./secrets/tls/ca.crt:/run/tls/ca.crt:ro + command: + - alembic + - upgrade + - head + depends_on: + postgres: + condition: service_healthy + tls-proxy: + condition: service_healthy + healthcheck: + disable: true + restart: 'no' + networks: + - backend + environment: + DATABASE_URL: ${DATABASE_URL} + TLS_CA_FILE: /run/tls/ca.crt + api: + deploy: + replicas: 2 + build: + context: . + dockerfile: Dockerfile + environment: + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: https://tls-proxy:14318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab + BAO_ENABLED: '1' + BAO_ADDR: https://openbao:8200 + BAO_CACERT: /run/tls/ca.crt + BAO_ROLE_ID_FILE: /run/secrets/bao/role_id + BAO_SECRET_ID_FILE: /run/secrets/bao/secret_id + BAO_SECRET_PATH: messenger/api + TLS_CA_FILE: /run/tls/ca.crt + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + S3_BUCKET: ${S3_BUCKET} + S3_REGION: ${S3_REGION:-us-east-1} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + ARGON2_MEMORY_KIB: ${ARGON2_MEMORY_KIB:-19456} + ARGON2_ITERATIONS: ${ARGON2_ITERATIONS:-2} + ARGON2_PARALLELISM: ${ARGON2_PARALLELISM:-1} + command: + - uvicorn + - app.main:app + - --http + - h11 + - --host + - 0.0.0.0 + - --port + - '8443' + - --ssl-certfile + - /run/tls/api.crt + - --ssl-keyfile + - /run/tls/api.key + - --ssl-ca-certs + - /run/tls/lb-ca.crt + - --ssl-cert-reqs + - '2' + - --timeout-keep-alive + - '5' + - --timeout-graceful-shutdown + - '20' + expose: + - '8443' + volumes: + - ./secrets/tls/ca.crt:/run/tls/ca.crt:ro + - ./secrets/tls/api.crt:/run/tls/api.crt:ro + - ./secrets/tls/api.key:/run/tls/api.key:ro + - ./secrets/tls/lb-ca.crt:/run/tls/lb-ca.crt:ro + - ./secrets/bao/api:/run/secrets/bao:ro + healthcheck: + test: + - CMD + - python + - -c + - import os; os.kill(1,0) + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + depends_on: + migrate: + condition: service_completed_successfully + seaweedfs: + condition: service_started + redis: + condition: service_healthy + otel-collector: + condition: service_started + openbao: + condition: service_started + tls-proxy: + condition: service_healthy + cpus: 1.0 + mem_limit: 768m + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + read_only: true + tmpfs: + - /tmp:size=64m,noexec,nosuid + restart: unless-stopped + networks: + - backend + outbox-publisher: + build: + context: . + dockerfile: Dockerfile + volumes: + - ./secrets/tls/ca.crt:/run/tls/ca.crt:ro + - ./secrets/bao/outbox:/run/secrets/bao:ro + command: + - python + - -m + - app.services.outbox_publisher + depends_on: + migrate: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + tls-proxy: + condition: service_healthy + openbao: + condition: service_started + healthcheck: + disable: true + restart: unless-stopped + networks: + - backend + environment: + BAO_ENABLED: '1' + BAO_ADDR: https://openbao:8200 + BAO_CACERT: /run/tls/ca.crt + BAO_ROLE_ID_FILE: /run/secrets/bao/role_id + BAO_SECRET_ID_FILE: /run/secrets/bao/secret_id + BAO_SECRET_PATH: messenger/outbox + TLS_CA_FILE: /run/tls/ca.crt + APP_ENV: ${APP_ENV:-lab} + OUTBOX_POLL_INTERVAL_SECONDS: ${OUTBOX_POLL_INTERVAL_SECONDS:-1} + OUTBOX_BATCH_SIZE: ${OUTBOX_BATCH_SIZE:-50} + cpus: 0.5 + mem_limit: 384m + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + read_only: true + tmpfs: + - /tmp:size=32m,noexec,nosuid + worker: + build: + context: . + dockerfile: Dockerfile + environment: + OTEL_SERVICE_NAME: messenger-worker + OTEL_EXPORTER_OTLP_ENDPOINT: https://tls-proxy:14318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab + BAO_ENABLED: '1' + BAO_ADDR: https://openbao:8200 + BAO_CACERT: /run/tls/ca.crt + BAO_ROLE_ID_FILE: /run/secrets/bao/role_id + BAO_SECRET_ID_FILE: /run/secrets/bao/secret_id + BAO_SECRET_PATH: messenger/worker + TLS_CA_FILE: /run/tls/ca.crt + APP_ENV: ${APP_ENV:-lab} + S3_BUCKET: ${S3_BUCKET} + S3_REGION: ${S3_REGION:-us-east-1} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_MAX_ATTEMPTS: ${WORKER_MAX_ATTEMPTS:-5} + WORKER_LEASE_SECONDS: ${WORKER_LEASE_SECONDS:-180} + WORKER_RETRY_BASE_SECONDS: ${WORKER_RETRY_BASE_SECONDS:-2} + WORKER_RETRY_MAX_SECONDS: ${WORKER_RETRY_MAX_SECONDS:-120} + WORKER_STAGING_RETENTION_SECONDS: ${WORKER_STAGING_RETENTION_SECONDS:-86400} + IMAGE_PROCESSING_TIMEOUT_SECONDS: ${IMAGE_PROCESSING_TIMEOUT_SECONDS:-60} + volumes: + - ./secrets/tls/ca.crt:/run/tls/ca.crt:ro + - ./secrets/bao/worker:/run/secrets/bao:ro + cpus: 1.0 + mem_limit: 1g + command: + - python + - -m + - app.worker + depends_on: + migrate: + condition: service_completed_successfully + seaweedfs: + condition: service_started + rabbitmq: + condition: service_healthy + otel-collector: + condition: service_started + openbao: + condition: service_started + tls-proxy: + condition: service_healthy + healthcheck: + disable: true + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + read_only: true + tmpfs: + - /tmp:size=128m,noexec,nosuid + restart: unless-stopped + networks: + - backend + nginx: + image: nginx:1.27.4-alpine + ports: + - 127.0.0.1:8443:8443 + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./nginx/proxy-common.conf:/etc/nginx/proxy-common.conf:ro + - ./secrets/tls/ca.crt:/run/tls/ca.crt:ro + - ./secrets/tls/nginx.crt:/run/tls/nginx.crt:ro + - ./secrets/tls/nginx.key:/run/tls/nginx.key:ro + - ./secrets/tls/lb-client.crt:/run/tls/lb-client.crt:ro + - ./secrets/tls/lb-client.key:/run/tls/lb-client.key:ro + depends_on: + api: + condition: service_started + healthcheck: + test: + - CMD-SHELL + - wget --quiet --no-check-certificate -O /dev/null https://127.0.0.1:8443/health/live || exit 1 + interval: 10s + timeout: 3s + retries: 10 + start_period: 10s + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + user: "101:101" + tmpfs: + - /var/cache/nginx:rw,noexec,nosuid,size=64m,uid=101,gid=101 + restart: unless-stopped + networks: + - backend + otel-collector: + image: otel/opentelemetry-collector-contrib:0.116.1 + command: + - --config=/etc/otelcol-contrib/config.yaml + volumes: + - ./observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro + - ./secrets/tls/ca.crt:/run/tls/ca.crt:ro + depends_on: + - prometheus + - loki + - tempo + restart: unless-stopped + networks: + - data_backend + prometheus: + image: prom/prometheus:v2.55.1 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + volumes: + - ./observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + - ./secrets/tls/ca.crt:/run/tls/ca.crt:ro + restart: unless-stopped + networks: + - data_backend + depends_on: + tls-proxy: + condition: service_healthy + loki: + image: grafana/loki:3.3.2 + command: + - -config.file=/etc/loki/local-config.yaml + volumes: + - ./observability/loki-config.yaml:/etc/loki/local-config.yaml:ro + - loki_data:/loki + restart: unless-stopped + networks: + - data_backend + tempo: + image: grafana/tempo:2.6.1 + command: + - -config.file=/etc/tempo/tempo.yaml + volumes: + - ./observability/tempo-config.yaml:/etc/tempo/tempo.yaml:ro + - tempo_data:/var/tempo + restart: unless-stopped + networks: + - data_backend + grafana: + image: grafana/grafana:11.4.0 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change_me} + GF_USERS_ALLOW_SIGN_UP: 'false' + volumes: + - grafana_data:/var/lib/grafana + - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro + - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - ./secrets/tls/ca.crt:/run/tls/ca.crt:ro + depends_on: + prometheus: + condition: service_started + loki: + condition: service_started + tempo: + condition: service_started + tls-proxy: + condition: service_healthy + restart: unless-stopped + networks: + - data_backend +volumes: + postgres_data: null + redis_data: null + rabbitmq_data: null + seaweedfs_data: null + prometheus_data: null + loki_data: null + tempo_data: null + grafana_data: null + openbao_data: null +networks: + backend: + driver: bridge + data_backend: + driver: bridge + internal: true diff --git a/contracts/trust-matrix.md b/contracts/trust-matrix.md index 0c2a977..3ed6e0b 100644 --- a/contracts/trust-matrix.md +++ b/contracts/trust-matrix.md @@ -1,37 +1,55 @@ -# Матрица доверия — заполнить в решении lab7 +# Матрица доверия — лабораторная 7 -Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать. +Матрица отражает фактически проверенную конфигурацию локального Docker Compose стенда. Если проверка конкретной связи не выполнялась отдельно, это прямо отмечено; наличие TLS proxy само по себе не считается доказательством правильной проверки identity или ACL. -| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг | -| --- | --- | --- | --- | -| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO | -| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO | -| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO | -| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO | -| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO | -| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO | -| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO | -| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO | -| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO | -| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO | -| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO | -| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO | +| Связь | Реализация / граница доверия | CA, identity и контроль | Положительная / отрицательная проверка | Доказательство / статус | +| --- | --- | --- | --- | --- | +| Клиент → ingress/LB | HTTPS через Nginx `127.0.0.1:8443` | Клиент доверяет `secrets/tls/ca.crt`; hostname `localhost`; прикладная авторизация Bearer | `tests/smoke.py` и `tests/security.py`: доверенный CA проходит; пустой trust store отклоняется | Nginx TLS ingress; smoke и security PASS | +| LB → API (все реплики) | Внутренний TLS/mTLS; API listener не опубликован на host | API проверяет клиентскую цепочку и URI SAN; разрешена `spiffe://lab7/ingress/nginx`; Nginx передаёт ingress identity | Без cert — TLS отказ; чужой CA — TLS отказ; доверенный `spiffe://lab7/worker` — отказ API; разрешённый ingress cert — успех | `app/main.py` identity allowlist; временный probe внутри API container; PASS | +| API/worker → PostgreSQL | Внутренняя связь через настроенную инфраструктуру TLS proxy/stunnel | PostgreSQL — внутренний Compose service. Точный режим TLS verification, DB role grants и сроки DB credentials в зафиксированных проверках отдельно не измерялись | Отдельный отрицательный тест сертификата/DB role не зафиксирован | `compose.yaml`, `infra/`; конфигурацию сверить перед заявлением полного verify-full/least privilege; не заявлять отдельный PASS | +| API/worker → S3 (SeaweedFS) | S3-compatible object storage внутри Compose; TLS proxy предусмотрен для внутренних связей | Раздельные сервисные настройки и storage boundary; конкретные denied-prefix проверки и срок действия ключей отдельно не зафиксированы | Smoke проверяет загрузку, скачивание и содержимое файла; отрицательный тест чужого prefix не зафиксирован | `app/services/object_storage.py`, `app/api/attachments.py`; функциональный smoke PASS, отдельный ACL-тест не заявлен | +| API/worker → broker (RabbitMQ) | Внутренний RabbitMQ; TLS proxy предусмотрен конфигурацией стенда | Отдельные producer/consumer ACL и SAN-проверки в имеющихся результатах не зафиксированы | Отдельный отрицательный тест broker identity/ACL не зафиксирован | `compose.yaml`, `infra/`; не заявлять mTLS/ACL PASS без отдельного вывода | +| API/worker → Collector | Telemetry экспортируется через конфигурацию telemetry/OTel | Конкретные TLS identity и отрицательная проверка Collector в имеющихся результатах не зафиксированы | Отдельный отрицательный тест ingest не зафиксирован | `infra/otel/`; требуется отдельное подтверждение | +| Collector → telemetry backends | Внутренняя telemetry инфраструктура | Точные TLS параметры и scoped credentials backend не подтверждены отдельным тестом | Отдельная отрицательная проверка не зафиксирована | `infra/otel/`; не заявлять отдельный PASS | +| Grafana → data sources | Не включено в подтверждённый набор проверок | TLS и identity Grafana/data sources отдельно не подтверждены | N/A для представленного набора доказательств; наличие компонента/подключения сверять с фактическим compose | Отдельный PASS не заявлен | +| API/worker/outbox → OpenBao | HTTPS к `https://openbao:8200`; CA проверяется из `/run/tls/ca.crt` | Отдельные AppRole identities и scoped policies для API, Worker, Outbox; root token не используется приложениями | TLS health запрос из API: HTTP 200; свой path: 200; чужой path: 403; запись: 403 | `infra/openbao/config.hcl`, policy templates, runtime env; все три AppRole policy checks PASS | +| Оператор → OpenBao admin | Host binding `127.0.0.1:18200` | TLS listener; операторская аутентификация. Root token/unseal material предназначены только для bootstrap/операторских действий | Приложения не используют root token; отдельный host admin TLS probe не засчитан из-за TLS EOF на host endpoint | OpenBao operator README и `compose.yaml`; административный доступ локальный; host TLS проверку не заявлять как PASS | +| API → Redis | Redis используется как внутренний сервис | TLS proxy предусмотрен для внутренних связей; точная проверка server identity/ACL отдельным отрицательным тестом не подтверждена | Отдельный тест Redis ACL/неверного сертификата не зафиксирован | `compose.yaml`, `infra/`; не заявлять отдельный PASS | +| Репликация/кластерные каналы | Используются внутренние Compose сети/механизмы сервисов | Отдельные проверки шифрования replication channels не зафиксированы | Отдельная проверка не выполнялась | Не заявлять отдельный PASS | +| Оператор → Grafana/S3 admin | Административные порты, если опубликованы, привязаны к loopback | Не предназначены для публичного ingress; конкретная аутентификация каждого admin UI отдельно не проверялась | Отдельный negative auth test не зафиксирован | Проверять по фактическим bindings в `compose.yaml`; не заявлять отдельный PASS | -Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход. +## Отрицательные проверки, выполненные на уровне 3 -## Обязательные отрицательные проверки +1. Внешний HTTPS: доверенный CA и hostname принимаются; пустой trust store отклоняется именно ошибкой проверки сертификата. +2. LB → API mTLS: отсутствие клиентского сертификата отклоняется на TLS handshake. +3. LB → API mTLS: сертификат от чужого CA отклоняется на TLS handshake. +4. LB → API identity: сертификат от доверенного CA с URI SAN `spiffe://lab7/worker` отклоняется API; разрешённая ingress identity `spiffe://lab7/ingress/nginx` принимается. +5. Пользователь вне чата не получает metadata/content вложения (404); поддельный `X-User-Id` не меняет Bearer identity. +6. OpenBao: AppRole может читать только собственный secret path; чтение чужого пути и запись запрещены (403). +7. Login rate limit: после установленного числа неудачных попыток возвращается 429 с положительным `Retry-After`. -1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен». -2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех. -3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity. -4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity. -5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3. -6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить. +Проверки broker/Collector identity, ротации/отзыва leaf certificates, отзыва SecretID и denied-prefix S3 не включены в подтверждённые результаты и не заявляются как выполненные. -## Bootstrap и эксплуатация +## Bootstrap и эксплуатация OpenBao -В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу. +- OpenBao работает в non-dev режиме (`openbao/openbao:2.2.0`) с persistent storage volume `openbao_data`. +- Конфигурация: `infra/openbao/config.hcl`; TLS listener и минимум TLS 1.2. +- API, Worker и Outbox используют разные AppRole credentials и scoped policies. +- TLS соединение API → OpenBao проверено с CA verification: запрос к `https://openbao:8200/v1/sys/health` с `verify=/run/tls/ca.crt` вернул HTTP 200. +- Policy-проверки выполнены для API, Worker и Outbox: собственный path — 200; чужой path — 403; запись — 403. +- Root token и unseal shares не должны попадать в приложение, Docker image, Git или отчёт. Bootstrap/unseal выполняет оператор. После обычного рестарта OpenBao состояние зависит от seal status; если OpenBao запечатан, оператор выполняет unseal с защищённо хранимыми shares. +- В приложениях не используется root token как постоянный `BAO_TOKEN`. +- Фактические TTL/renewal параметры AppRole и процедура ротации должны сверяться с policy/role конфигурацией перед отдельным заявлением о полном lifecycle credentials. -OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии. +## Сетевые границы и контейнеры -Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap. +- Nginx публикует внешний HTTPS только на `127.0.0.1:8443`. +- API не публикует бизнес-порт напрямую на host; внутренний API listener требует mTLS. +- OpenBao admin endpoint привязан к `127.0.0.1:18200`. +- Вспомогательные TLS proxy ports привязаны к loopback. +- Проверка `docker inspect`: API/Worker/Outbox работают не от root, `read_only=true`, `CapDrop=ALL`, `privileged=false`, имеют CPU/RAM limits. +- Проверка compose/Dockerfile не обнаружила `docker.sock`, `network_mode: host` или `privileged: true`. + +## Не заявленные проверки + +В рамках целевой оценки 3 не заявляются как завершённые: mTLS/ACL negative tests для RabbitMQ и Collector; denied-prefix тест S3; отдельная проверка TLS verify-full и DB role grants PostgreSQL; отзыв/ротация сертификатов и AppRole SecretID; проверка Grafana datasource identity; полный аудит всех внутренних каналов. Их отсутствие в отчёте не следует трактовать как успешное прохождение. diff --git a/grafana/dashboards/messenger-overview.json b/grafana/dashboards/messenger-overview.json new file mode 100644 index 0000000..07c1658 --- /dev/null +++ b/grafana/dashboards/messenger-overview.json @@ -0,0 +1,171 @@ +{ + "id": null, + "uid": "messenger-overview", + "title": "Messenger — Observability", + "tags": ["messenger", "lab6", "observability"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "API Requests Per Second", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_http_server_duration_milliseconds_count[5m]))", + "legendFormat": "Requests/sec" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": {"displayMode": "list", "placement": "bottom"}, + "tooltip": {"mode": "multi"} + } + }, + { + "id": 2, + "title": "API Error Rate", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "refId": "A", + "expr": "100 * sum(rate(messenger_http_server_duration_milliseconds_count{http_response_status_code=~\"5..\"}[5m])) / clamp_min(sum(rate(messenger_http_server_duration_milliseconds_count[5m])), 0.001)", + "legendFormat": "5xx errors (%)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": {"displayMode": "list", "placement": "bottom"}, + "tooltip": {"mode": "multi"} + } + }, + { + "id": 3, + "title": "API Request Duration — p95", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_server_duration_milliseconds_bucket[5m])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": {"displayMode": "list", "placement": "bottom"}, + "tooltip": {"mode": "multi"} + } + }, + { + "id": 4, + "title": "Successful Background Jobs", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_jobs_processed_total[5m]))", + "legendFormat": "Successful jobs/sec" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": {"displayMode": "list", "placement": "bottom"}, + "tooltip": {"mode": "multi"} + } + }, + { + "id": 5, + "title": "Failed Background Jobs", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_jobs_failed_total[5m]))", + "legendFormat": "Failed jobs/sec" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": {"displayMode": "list", "placement": "bottom"}, + "tooltip": {"mode": "multi"} + } + }, + { + "id": 6, + "title": "Job Processing Duration — p95", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": {"displayMode": "list", "placement": "bottom"}, + "tooltip": {"mode": "multi"} + } + } + ], + "annotations": { + "list": [] + }, + "links": [] +} diff --git a/grafana/provisioning/dashboards/dashboards.yaml b/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..4c10dd4 --- /dev/null +++ b/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: Messenger + orgId: 1 + folder: Messenger + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards diff --git a/grafana/provisioning/datasources/datasources.yaml b/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..43d1d7c --- /dev/null +++ b/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,41 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: https://tls-proxy:19090 + isDefault: true + editable: false + jsonData: + tlsAuthWithCACert: true + secureJsonData: + tlsCACert: "$__file{/run/tls/ca.crt}" + + - name: Loki + uid: loki + type: loki + access: proxy + url: https://tls-proxy:13100 + editable: false + jsonData: + tlsAuthWithCACert: true + derivedFields: + - name: TraceID + matcherRegex: '"trace_id":"([0-9a-f]{32})"' + datasourceUid: tempo + url: '$${__value.raw}' + secureJsonData: + tlsCACert: "$__file{/run/tls/ca.crt}" + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: https://tls-proxy:13200 + editable: false + jsonData: + tlsAuthWithCACert: true + secureJsonData: + tlsCACert: "$__file{/run/tls/ca.crt}" diff --git a/infra/openbao/README.md b/infra/openbao/README.md index 0850e23..1f2876a 100644 --- a/infra/openbao/README.md +++ b/infra/openbao/README.md @@ -1,7 +1,25 @@ -# OpenBao — реализовать конфигурацию и политики +# OpenBao bootstrap (non-dev, persistent) -Добавьте non-dev конфигурацию listener/storage/TLS, API/worker policies, bootstrap и rotation scripts. Credentials и unseal material создаются локально; здесь нет готового root token или private key. +OpenBao uses file storage in the `openbao_data` volume and HTTPS on 8200. The operator owns initialization, unseal shares and the initial root token. They are never stored in Git, `.env`, images or application configuration. -Решение должно показать реальную сервисную аутентификацию и получение нужного secret по ограниченной policy; просто работающий контейнер OpenBao не засчитывается. На 3 статический KV secret допустим. На 4 покажите expiry/renewal/rotation и allow/deny. Путь KV v2 data/metadata и ACL должен соответствовать фактическому mount; не копируйте широкое `path "*"` для приложения. +## First initialization (PowerShell) -Документируйте шаги оператора init/unseal и восстановление после restart. Скрипт не должен печатать tokens/shares или хранить их в Git. См. contracts/trust-matrix.md и REPORT.md. +1. Generate local TLS files with `scripts/bootstrap-certs.ps1`. +2. Start only OpenBao: `docker compose up -d openbao`. +3. Initialize once and save the output securely offline: + + `docker compose exec openbao bao operator init -key-shares=3 -key-threshold=2` + +4. Unseal using two distinct shares: + + `docker compose exec openbao bao operator unseal` + + Repeat with the second share. Never put shares in scripts or source control. +5. Run `scripts/bootstrap-openbao.ps1`. It prompts for the root token without echoing it, creates KV v2, policies and AppRole roles, then writes short-lived SecretIDs and RoleIDs to ignored local files under `secrets/bao/`. +6. Start API/worker after bootstrap. Each service authenticates with its own AppRole and reads only its own KV path. The application never receives the root token or unseal material. + +## Restart + +The storage volume persists. After an ordinary restart, check `bao status`. If sealed, an operator must unseal with two shares. Do not automate unseal by placing shares beside the container. + +KV v2 secrets are static for this lab-3 baseline. Rotate AppRole SecretIDs by generating new IDs, updating the local files and restarting the affected service. Dynamic credentials and zero-downtime rotation are outside the selected grade. diff --git a/infra/openbao/api-policy.hcl b/infra/openbao/api-policy.hcl new file mode 100644 index 0000000..190c529 --- /dev/null +++ b/infra/openbao/api-policy.hcl @@ -0,0 +1,3 @@ +path "secret/data/messenger/api" { + capabilities = ["read"] +} diff --git a/infra/openbao/config.hcl b/infra/openbao/config.hcl new file mode 100644 index 0000000..5e43e67 --- /dev/null +++ b/infra/openbao/config.hcl @@ -0,0 +1,16 @@ +ui = false +api_addr = "https://openbao:8200" +cluster_addr = "https://openbao:8201" + +storage "file" { + path = "/openbao/data" +} + +listener "tcp" { + address = "0.0.0.0:8200" + cluster_address = "0.0.0.0:8201" + tls_disable = 0 + tls_cert_file = "/run/tls/openbao.crt" + tls_key_file = "/run/tls/openbao.key" + tls_min_version = "tls12" +} diff --git a/infra/openbao/outbox-policy.hcl b/infra/openbao/outbox-policy.hcl new file mode 100644 index 0000000..a4b82b1 --- /dev/null +++ b/infra/openbao/outbox-policy.hcl @@ -0,0 +1,3 @@ +path "secret/data/messenger/outbox" { + capabilities = ["read"] +} diff --git a/infra/openbao/worker-policy.hcl b/infra/openbao/worker-policy.hcl new file mode 100644 index 0000000..6c67e26 --- /dev/null +++ b/infra/openbao/worker-policy.hcl @@ -0,0 +1,3 @@ +path "secret/data/messenger/worker" { + capabilities = ["read"] +} diff --git a/infra/stunnel/stunnel.conf b/infra/stunnel/stunnel.conf new file mode 100644 index 0000000..d218719 --- /dev/null +++ b/infra/stunnel/stunnel.conf @@ -0,0 +1,78 @@ +foreground = yes +pid = +output = /dev/stdout +debug = 4 +sslVersionMin = TLSv1.2 + +[postgres] +accept = 0.0.0.0:15432 +connect = postgres:5432 +protocol = pgsql +cert = /cert.crt +key = /private.key + +[redis] +accept = 0.0.0.0:16379 +connect = redis:6379 +cert = /cert.crt +key = /private.key + +[rabbitmq] +accept = 0.0.0.0:15672 +connect = rabbitmq:5672 +cert = /cert.crt +key = /private.key + +[s3] +accept = 0.0.0.0:18333 +connect = seaweedfs:8333 +cert = /cert.crt +key = /private.key + +[s3-public] +accept = 0.0.0.0:18443 +connect = seaweedfs:8333 +cert = /cert.crt +key = /private.key + +[otlp] +accept = 0.0.0.0:14318 +connect = otel-collector:4318 +cert = /cert.crt +key = /private.key + +[loki] +accept = 0.0.0.0:13100 +connect = loki:3100 +cert = /cert.crt +key = /private.key + +[tempo] +accept = 0.0.0.0:14320 +connect = tempo:4318 +cert = /cert.crt +key = /private.key + +[prometheus] +accept = 0.0.0.0:19090 +connect = prometheus:9090 +cert = /cert.crt +key = /private.key + +[otel-metrics] +accept = 0.0.0.0:18889 +connect = otel-collector:8889 +cert = /cert.crt +key = /private.key + +[grafana] +accept = 0.0.0.0:13001 +connect = grafana:3000 +cert = /cert.crt +key = /private.key + +[tempo-query] +accept = 0.0.0.0:13200 +connect = tempo:3200 +cert = /cert.crt +key = /private.key diff --git a/migrations/README b/migrations/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..0bc520a --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,60 @@ +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +from app.core.config import settings +from app.db.base import Base +from app.db import models + + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +config.set_main_option( + "sqlalchemy.url", + settings.database_url.replace("%", "%%"), +) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + url = settings.database_url + + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + compare_type=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() \ No newline at end of file diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/21ad89656327_create_initial_database_schema.py b/migrations/versions/21ad89656327_create_initial_database_schema.py new file mode 100644 index 0000000..db445ba --- /dev/null +++ b/migrations/versions/21ad89656327_create_initial_database_schema.py @@ -0,0 +1,79 @@ +"""create initial database schema + +Revision ID: 21ad89656327 +Revises: +Create Date: 2026-11-04 17:31:39.336368 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '21ad89656327' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('chats', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('title', sa.String(length=100), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('users', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('username', sa.String(length=32), nullable=False), + sa.Column('display_name', sa.String(length=100), nullable=False), + sa.Column('password_hash', sa.String(length=255), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_users_username'), 'users', ['username'], unique=True) + op.create_table('chat_members', + sa.Column('chat_id', sa.UUID(), nullable=False), + sa.Column('user_id', sa.UUID(), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('chat_id', 'user_id') + ) + op.create_table('messages', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('chat_id', sa.UUID(), nullable=False), + sa.Column('sender_id', sa.UUID(), nullable=False), + sa.Column('text', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['sender_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('ix_messages_chat_created_id', 'messages', ['chat_id', 'created_at', 'id'], unique=False) + op.create_table('sessions', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('token_hash', sa.String(length=64), nullable=False), + sa.Column('user_id', sa.UUID(), nullable=False), + sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_sessions_token_hash'), 'sessions', ['token_hash'], unique=True) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_sessions_token_hash'), table_name='sessions') + op.drop_table('sessions') + op.drop_index('ix_messages_chat_created_id', table_name='messages') + op.drop_table('messages') + op.drop_table('chat_members') + op.drop_index(op.f('ix_users_username'), table_name='users') + op.drop_table('users') + op.drop_table('chats') + # ### end Alembic commands ### diff --git a/migrations/versions/4d9a1f2c7b10_add_attachments.py b/migrations/versions/4d9a1f2c7b10_add_attachments.py new file mode 100644 index 0000000..c29c05f --- /dev/null +++ b/migrations/versions/4d9a1f2c7b10_add_attachments.py @@ -0,0 +1,37 @@ +"""add attachment metadata""" +from alembic import op +import sqlalchemy as sa +from typing import Sequence, Union + +revision: str = "4d9a1f2c7b10" +down_revision: Union[str, None] = "8c3d9f4a21b7" +branch_labels = None +depends_on = None + +def upgrade() -> None: + op.add_column("messages", sa.Column("attachment_ids", sa.JSON(), nullable=False, server_default="[]")) + op.create_table( + "attachments", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("chat_id", sa.UUID(), nullable=False), + sa.Column("owner_id", sa.UUID(), nullable=False), + sa.Column("filename", sa.String(255), nullable=False), + sa.Column("content_type", sa.String(32), nullable=False), + sa.Column("size_bytes", sa.Integer(), nullable=False), + sa.Column("sha256", sa.String(64), nullable=False), + sa.Column("object_key", sa.String(512), nullable=False), + sa.Column("status", sa.String(16), nullable=False), + sa.Column("error_code", sa.String(64), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["owner_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("object_key"), + ) + op.create_index("ix_attachments_chat_id", "attachments", ["chat_id"]) + +def downgrade() -> None: + op.drop_index("ix_attachments_chat_id", table_name="attachments") + op.drop_table("attachments") + op.drop_column("messages", "attachment_ids") diff --git a/migrations/versions/8c3d9f4a21b7_add_idempotency_records.py b/migrations/versions/8c3d9f4a21b7_add_idempotency_records.py new file mode 100644 index 0000000..a228b42 --- /dev/null +++ b/migrations/versions/8c3d9f4a21b7_add_idempotency_records.py @@ -0,0 +1,39 @@ +"""add shared PostgreSQL idempotency records + +Revision ID: 8c3d9f4a21b7 +Revises: 21ad89656327 +Create Date: 2026-11-05 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "8c3d9f4a21b7" +down_revision: Union[str, None] = "21ad89656327" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "idempotency_records", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("user_id", sa.UUID(), nullable=False), + sa.Column("method", sa.String(length=10), nullable=False), + sa.Column("path", sa.String(length=255), nullable=False), + sa.Column("idempotency_key", sa.String(length=128), nullable=False), + sa.Column("request_hash", sa.String(length=64), nullable=False), + sa.Column("message_id", sa.UUID(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id", "method", "path", "idempotency_key", name="uq_idempotency_scope_key"), + ) + op.create_index("ix_idempotency_records_expires_at", "idempotency_records", ["expires_at"], unique=False) + + +def downgrade() -> None: + op.drop_index("ix_idempotency_records_expires_at", table_name="idempotency_records") + op.drop_table("idempotency_records") diff --git a/migrations/versions/a7c2e91d5f40_add_attachment_uploads.py b/migrations/versions/a7c2e91d5f40_add_attachment_uploads.py new file mode 100644 index 0000000..4917219 --- /dev/null +++ b/migrations/versions/a7c2e91d5f40_add_attachment_uploads.py @@ -0,0 +1,78 @@ +"""add attachment uploads + +Revision ID: a7c2e91d5f40 +Revises: 4d9a1f2c7b10 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "a7c2e91d5f40" +down_revision: Union[str, None] = "4d9a1f2c7b10" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "attachment_uploads", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("chat_id", sa.UUID(), nullable=False), + sa.Column("owner_id", sa.UUID(), nullable=False), + sa.Column("filename", sa.String(length=255), nullable=False), + sa.Column("content_type", sa.String(length=32), nullable=False), + sa.Column("expected_size_bytes", sa.Integer(), nullable=False), + sa.Column("expected_sha256", sa.String(length=64), nullable=False), + sa.Column("object_key", sa.String(length=512), nullable=False), + sa.Column("status", sa.String(length=16), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["chat_id"], + ["chats.id"], + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["owner_id"], + ["users.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("object_key"), + ) + + op.create_index( + "ix_attachment_uploads_chat_id", + "attachment_uploads", + ["chat_id"], + ) + op.create_index( + "ix_attachment_uploads_owner_id", + "attachment_uploads", + ["owner_id"], + ) + op.create_index( + "ix_attachment_uploads_expires_at", + "attachment_uploads", + ["expires_at"], + ) + + +def downgrade() -> None: + op.drop_index( + "ix_attachment_uploads_expires_at", + table_name="attachment_uploads", + ) + op.drop_index( + "ix_attachment_uploads_owner_id", + table_name="attachment_uploads", + ) + op.drop_index( + "ix_attachment_uploads_chat_id", + table_name="attachment_uploads", + ) + op.drop_table("attachment_uploads") \ No newline at end of file diff --git a/migrations/versions/a9d670bf1c16_add_durable_image_processing_jobs.py b/migrations/versions/a9d670bf1c16_add_durable_image_processing_jobs.py new file mode 100644 index 0000000..a705228 --- /dev/null +++ b/migrations/versions/a9d670bf1c16_add_durable_image_processing_jobs.py @@ -0,0 +1,210 @@ + +"""add durable image processing jobs""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "a9d670bf1c16" +down_revision: Union[str, None] = "a7c2e91d5f40" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Дополнительные поля для хранения исходника и версии конвейера. + op.add_column( + "attachments", + sa.Column("staging_key", sa.String(length=512), nullable=True), + ) + op.create_unique_constraint( + "uq_attachments_staging_key", + "attachments", + ["staging_key"], + ) + op.add_column( + "attachments", + sa.Column( + "pipeline_version", + sa.Integer(), + nullable=False, + server_default="1", + ), + ) + + # Задания на асинхронную обработку. + op.create_table( + "processing_jobs", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("attachment_id", sa.UUID(), nullable=False), + sa.Column( + "pipeline_version", + sa.Integer(), + nullable=False, + server_default="1", + ), + sa.Column( + "attempt", + sa.Integer(), + nullable=False, + server_default="1", + ), + sa.Column( + "status", + sa.String(length=16), + nullable=False, + server_default="queued", + ), + sa.Column( + "available_at", + sa.DateTime(timezone=True), + nullable=False, + ), + sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True), + sa.Column("worker_id", sa.String(length=128), nullable=True), + sa.Column( + "last_error_code", + sa.String(length=64), + nullable=True, + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["attachment_id"], + ["attachments.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "attachment_id", + "pipeline_version", + name="uq_processing_job_attachment_pipeline", + ), + ) + op.create_index( + "ix_processing_jobs_status_available", + "processing_jobs", + ["status", "available_at"], + ) + op.create_index( + "ix_processing_jobs_lease_until", + "processing_jobs", + ["lease_until"], + ) + + # Созданные варианты изображений. + op.create_table( + "attachment_variants", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("attachment_id", sa.UUID(), nullable=False), + sa.Column("variant", sa.String(length=32), nullable=False), + sa.Column("object_key", sa.String(length=512), nullable=False), + sa.Column("content_type", sa.String(length=32), nullable=False), + sa.Column("width", sa.Integer(), nullable=False), + sa.Column("height", sa.Integer(), nullable=False), + sa.Column("size_bytes", sa.Integer(), nullable=False), + sa.Column("sha256", sa.String(length=64), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["attachment_id"], + ["attachments.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("object_key"), + sa.UniqueConstraint( + "attachment_id", + "variant", + name="uq_attachment_variant", + ), + ) + op.create_index( + "ix_attachment_variants_attachment_id", + "attachment_variants", + ["attachment_id"], + ) + + # Transactional outbox: события публикуются в брокер + # отдельным процессом после фиксации транзакции. + op.create_table( + "outbox_events", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("event_type", sa.String(length=128), nullable=False), + sa.Column("aggregate_id", sa.UUID(), nullable=False), + sa.Column("payload", sa.JSON(), nullable=False), + sa.Column( + "status", + sa.String(length=16), + nullable=False, + server_default="pending", + ), + sa.Column( + "attempts", + sa.Integer(), + nullable=False, + server_default="0", + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + ), + sa.Column( + "published_at", + sa.DateTime(timezone=True), + nullable=True, + ), + sa.Column("last_error", sa.String(length=256), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_outbox_events_status_created", + "outbox_events", + ["status", "created_at"], + ) + + +def downgrade() -> None: + op.drop_index( + "ix_outbox_events_status_created", + table_name="outbox_events", + ) + op.drop_table("outbox_events") + + op.drop_index( + "ix_attachment_variants_attachment_id", + table_name="attachment_variants", + ) + op.drop_table("attachment_variants") + + op.drop_index( + "ix_processing_jobs_lease_until", + table_name="processing_jobs", + ) + op.drop_index( + "ix_processing_jobs_status_available", + table_name="processing_jobs", + ) + op.drop_table("processing_jobs") + + op.drop_column("attachments", "pipeline_version") + op.drop_constraint( + "uq_attachments_staging_key", + "attachments", + type_="unique", + ) + op.drop_column("attachments", "staging_key") diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..846456a --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,40 @@ +worker_processes auto; +pid /tmp/nginx.pid; +error_log /dev/stderr warn; + +events { + worker_connections 1024; +} + +http { + access_log /dev/stdout; + server_tokens off; + + resolver 127.0.0.11 valid=10s ipv6=off; + + client_body_temp_path /var/cache/nginx/client_temp; + proxy_temp_path /var/cache/nginx/proxy_temp; + + client_max_body_size 10m; + + upstream api_backend { + zone api_backend 64k; + server api:8443 resolve; + } + + server { + listen 8443 ssl; + server_name localhost; + + ssl_certificate /run/tls/nginx.crt; + ssl_certificate_key /run/tls/nginx.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 10m; + + location / { + proxy_pass https://api_backend; + include /etc/nginx/proxy-common.conf; + } + } +} diff --git a/nginx/proxy-common.conf b/nginx/proxy-common.conf new file mode 100644 index 0000000..72f733c --- /dev/null +++ b/nginx/proxy-common.conf @@ -0,0 +1,20 @@ +proxy_http_version 1.1; +proxy_set_header Host $host; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto https; +proxy_set_header X-Lab7-Client-Identity "spiffe://lab7/ingress/nginx"; +proxy_set_header Connection ""; +proxy_connect_timeout 5s; +proxy_send_timeout 120s; +proxy_read_timeout 120s; + +proxy_ssl_certificate /run/tls/lb-client.crt; +proxy_ssl_certificate_key /run/tls/lb-client.key; +proxy_ssl_trusted_certificate /run/tls/ca.crt; +proxy_ssl_verify on; +proxy_ssl_verify_depth 2; +proxy_ssl_server_name on; +proxy_ssl_name api; +proxy_ssl_protocols TLSv1.2 TLSv1.3; +proxy_next_upstream error timeout http_502 http_503 http_504; diff --git a/observability/loki-config.yaml b/observability/loki-config.yaml new file mode 100644 index 0000000..c7f1639 --- /dev/null +++ b/observability/loki-config.yaml @@ -0,0 +1,35 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem diff --git a/observability/otel-collector-config.yaml b/observability/otel-collector-config.yaml new file mode 100644 index 0000000..695ca35 --- /dev/null +++ b/observability/otel-collector-config.yaml @@ -0,0 +1,52 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 5s + limit_mib: 256 + spike_limit_mib: 64 + batch: + timeout: 5s + send_batch_size: 1024 + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + namespace: messenger + send_timestamps: true + + otlphttp/tempo: + endpoint: https://tls-proxy:14320 + tls: + ca_file: /run/tls/ca.crt + + otlphttp/loki: + endpoint: https://tls-proxy:13100/otlp + tls: + ca_file: /run/tls/ca.crt + + debug: + verbosity: basic + +service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo, debug] + + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus, debug] + + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki, debug] diff --git a/observability/prometheus.yml b/observability/prometheus.yml new file mode 100644 index 0000000..0574e92 --- /dev/null +++ b/observability/prometheus.yml @@ -0,0 +1,11 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: otel-collector + scheme: https + tls_config: + ca_file: /run/tls/ca.crt + static_configs: + - targets: ["tls-proxy:18889"] diff --git a/observability/tempo-config.yaml b/observability/tempo-config.yaml new file mode 100644 index 0000000..ec204f9 --- /dev/null +++ b/observability/tempo-config.yaml @@ -0,0 +1,31 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +overrides: + defaults: + metrics_generator: + processors: [] diff --git a/postgres/init-roles.sh b/postgres/init-roles.sh new file mode 100644 index 0000000..d88abc4 --- /dev/null +++ b/postgres/init-roles.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu + +: "${API_DB_USER:?API_DB_USER is required}" +: "${API_DB_PASSWORD:?API_DB_PASSWORD is required}" +: "${WORKER_DB_USER:?WORKER_DB_USER is required}" +: "${WORKER_DB_PASSWORD:?WORKER_DB_PASSWORD is required}" +: "${OUTBOX_DB_USER:?OUTBOX_DB_USER is required}" +: "${OUTBOX_DB_PASSWORD:?OUTBOX_DB_PASSWORD is required}" + +psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" --set ON_ERROR_STOP=1 \ + --set api_user="$API_DB_USER" --set api_password="$API_DB_PASSWORD" \ + --set worker_user="$WORKER_DB_USER" --set worker_password="$WORKER_DB_PASSWORD" \ + --set outbox_user="$OUTBOX_DB_USER" --set outbox_password="$OUTBOX_DB_PASSWORD" <<'SQL' +SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'api_user', :'api_password') \gexec +SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'worker_user', :'worker_password') \gexec +SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'outbox_user', :'outbox_password') \gexec +SELECT format('GRANT CONNECT ON DATABASE %I TO %I', current_database(), :'api_user') \gexec +SELECT format('GRANT CONNECT ON DATABASE %I TO %I', current_database(), :'worker_user') \gexec +SELECT format('GRANT CONNECT ON DATABASE %I TO %I', current_database(), :'outbox_user') \gexec +GRANT USAGE ON SCHEMA public TO :"api_user", :"worker_user", :"outbox_user"; +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO :"api_user", :"worker_user"; +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT USAGE, SELECT ON SEQUENCES TO :"api_user", :"worker_user"; +SQL diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3023623 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,19 @@ +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +pydantic==2.10.4 +pydantic-settings==2.7.1 +SQLAlchemy==2.0.36 +psycopg[binary]==3.2.3 +alembic==1.14.0 +redis==5.2.1 +pwdlib[argon2]==0.2.1 +boto3==1.35.99 +Pillow==11.0.0 +aio-pika==9.5.5 +opentelemetry-api==1.29.0 +opentelemetry-sdk==1.29.0 +opentelemetry-exporter-otlp-proto-http==1.29.0 +opentelemetry-instrumentation-fastapi==0.50b0 +opentelemetry-instrumentation-sqlalchemy==0.50b0 +opentelemetry-instrumentation-logging==0.50b0 +requests==2.32.3 diff --git a/scripts/bootstrap-certs.ps1 b/scripts/bootstrap-certs.ps1 new file mode 100644 index 0000000..63120f2 --- /dev/null +++ b/scripts/bootstrap-certs.ps1 @@ -0,0 +1,107 @@ +$ErrorActionPreference = "Stop" + +$Root = Split-Path -Parent $PSScriptRoot +$Tls = Join-Path $Root "secrets/tls" + +New-Item -ItemType Directory -Force -Path $Tls | Out-Null + +$OpenSSL = Get-Command openssl -ErrorAction SilentlyContinue +if (-not $OpenSSL) { + throw "OpenSSL not found in PATH" +} + +function Run-OpenSSL([string[]]$OpenSslArgs) { + & openssl @OpenSslArgs + if ($LASTEXITCODE -ne 0) { + throw "OpenSSL failed: $OpenSslArgs" + } +} + +if (-not (Test-Path "$Tls/ca.key")) { + Run-OpenSSL @( + "req", "-x509", "-newkey", "rsa:3072", + "-sha256", "-days", "3650", "-nodes", + "-keyout", "$Tls/ca.key", + "-out", "$Tls/ca.crt", + "-subj", "/CN=Lab7 Server Root CA", + "-addext", "basicConstraints=critical,CA:TRUE", + "-addext", "keyUsage=critical,keyCertSign,cRLSign" + ) +} + +if (-not (Test-Path "$Tls/lb-ca.key")) { + Run-OpenSSL @( + "req", "-x509", "-newkey", "rsa:3072", + "-sha256", "-days", "3650", "-nodes", + "-keyout", "$Tls/lb-ca.key", + "-out", "$Tls/lb-ca.crt", + "-subj", "/CN=Lab7 Ingress Client CA", + "-addext", "basicConstraints=critical,CA:TRUE", + "-addext", "keyUsage=critical,keyCertSign,cRLSign" + ) +} + +function New-ServerCert([string]$Name, [string]$San) { + $key = "$Tls/$Name.key" + $csr = "$Tls/$Name.csr" + $crt = "$Tls/$Name.crt" + $ext = "$Tls/$Name.ext" + + if (Test-Path $crt) { + return + } + + Set-Content -Encoding ascii $ext "basicConstraints=critical,CA:FALSE`nsubjectAltName=$San`nextendedKeyUsage=serverAuth`nkeyUsage=digitalSignature,keyEncipherment" + + Run-OpenSSL @( + "req", "-new", "-newkey", "rsa:2048", "-nodes", + "-keyout", $key, + "-out", $csr, + "-subj", "/CN=$Name" + ) + + Run-OpenSSL @( + "x509", "-req", "-in", $csr, + "-CA", "$Tls/ca.crt", + "-CAkey", "$Tls/ca.key", + "-CAcreateserial", + "-out", $crt, + "-days", "825", + "-sha256", + "-extfile", $ext + ) + + Remove-Item $csr, $ext -Force -ErrorAction SilentlyContinue +} + +New-ServerCert "nginx" "DNS:localhost,IP:127.0.0.1" +New-ServerCert "api" "DNS:api,DNS:localhost,IP:127.0.0.1" +New-ServerCert "openbao" "DNS:openbao,DNS:localhost,IP:127.0.0.1" +New-ServerCert "tls-proxy" "DNS:tls-proxy,DNS:localhost,IP:127.0.0.1" + +if (-not (Test-Path "$Tls/lb-client.crt")) { + Set-Content -Encoding ascii "$Tls/lb-client.ext" "basicConstraints=critical,CA:FALSE`nsubjectAltName=URI:spiffe://lab7/ingress/nginx`nextendedKeyUsage=clientAuth`nkeyUsage=digitalSignature,keyEncipherment" + + Run-OpenSSL @( + "req", "-new", "-newkey", "rsa:2048", "-nodes", + "-keyout", "$Tls/lb-client.key", + "-out", "$Tls/lb-client.csr", + "-subj", "/CN=lab7-nginx/OU=ingress" + ) + + Run-OpenSSL @( + "x509", "-req", "-in", "$Tls/lb-client.csr", + "-CA", "$Tls/lb-ca.crt", + "-CAkey", "$Tls/lb-ca.key", + "-CAcreateserial", + "-out", "$Tls/lb-client.crt", + "-days", "365", + "-sha256", + "-extfile", "$Tls/lb-client.ext" + ) + + Remove-Item "$Tls/lb-client.csr", "$Tls/lb-client.ext" -Force -ErrorAction SilentlyContinue +} + +Write-Host "Certificates created under secrets/tls." +Write-Host "Private keys must never be committed to Git." diff --git a/scripts/bootstrap-openbao.ps1 b/scripts/bootstrap-openbao.ps1 new file mode 100644 index 0000000..667e14c --- /dev/null +++ b/scripts/bootstrap-openbao.ps1 @@ -0,0 +1,114 @@ +$ErrorActionPreference = "Stop" +$Root = Split-Path -Parent $PSScriptRoot +$Tls = Join-Path $Root "secrets/tls" +$BaoDir = Join-Path $Root "secrets/bao" +$EnvFile = Join-Path $Root ".env" +if (-not (Test-Path $EnvFile)) { throw "Create .env from .env.example first." } +$vars = @{} +Get-Content $EnvFile | ForEach-Object { + $line = $_.Trim() + if ($line -and -not $line.StartsWith("#") -and $line.Contains("=")) { + $pair = $line.Split("=",2) + $vars[$pair[0].Trim()] = $pair[1].Trim().Trim('"').Trim("'") + } +} +if (-not $vars.POSTGRES_USER -or -not $vars.POSTGRES_PASSWORD -or -not $vars.POSTGRES_DB -or + -not $vars.API_DB_USER -or -not $vars.API_DB_PASSWORD -or + -not $vars.WORKER_DB_USER -or -not $vars.WORKER_DB_PASSWORD -or + -not $vars.OUTBOX_DB_USER -or -not $vars.OUTBOX_DB_PASSWORD -or + -not $vars.PUBLISHER_RABBITMQ_USER -or -not $vars.PUBLISHER_RABBITMQ_PASSWORD -or + -not $vars.WORKER_RABBITMQ_USER -or -not $vars.WORKER_RABBITMQ_PASSWORD -or + -not $vars.REDIS_ADMIN_PASSWORD -or -not $vars.API_REDIS_PASSWORD -or -not $vars.WORKER_REDIS_PASSWORD) { + throw ".env must define admin, API, Worker and Outbox database/broker credentials." +} +Import-Certificate -FilePath "$Tls/ca.crt" -CertStoreLocation Cert:\CurrentUser\Root | Out-Null +$rootSecure = Read-Host "OpenBao operator root token (input is hidden by PowerShell secure prompt)" -AsSecureString +$ptr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($rootSecure) +try { $rootToken = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($ptr) } +finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($ptr) } +$base = "https://localhost:18200/v1" +$headers = @{ "X-Vault-Token" = $rootToken } +function BaoPut($path,$body) { + Invoke-RestMethod -Method Put -Uri "$base/$path" -Headers $headers -ContentType "application/json" -Body ($body | ConvertTo-Json -Depth 12) | Out-Null +} +function BaoPost($path,$body) { + Invoke-RestMethod -Method Post -Uri "$base/$path" -Headers $headers -ContentType "application/json" -Body ($body | ConvertTo-Json -Depth 12) +} +# KV v2 mount and AppRole auth. +try { BaoPost "sys/mounts/secret" @{type="kv"; options=@{version="2"}} } catch {} +try { BaoPost "sys/auth/approle" @{type="approle"} } catch {} +$apiPolicy = [System.IO.File]::ReadAllText("$Root/infra/openbao/api-policy.hcl") +$workerPolicy = [System.IO.File]::ReadAllText("$Root/infra/openbao/worker-policy.hcl") +$outboxPolicy = [System.IO.File]::ReadAllText("$Root/infra/openbao/outbox-policy.hcl") +BaoPut "sys/policies/acl/api-policy" @{policy=$apiPolicy} +BaoPut "sys/policies/acl/worker-policy" @{policy=$workerPolicy} +BaoPut "sys/policies/acl/outbox-policy" @{policy=$outboxPolicy} + +$apiDb = "postgresql+psycopg://$($vars.API_DB_USER):$($vars.API_DB_PASSWORD)@tls-proxy:15432/$($vars.POSTGRES_DB)?sslmode=verify-full&sslrootcert=/run/tls/ca.crt" +$workerDb = "postgresql+psycopg://$($vars.WORKER_DB_USER):$($vars.WORKER_DB_PASSWORD)@tls-proxy:15432/$($vars.POSTGRES_DB)?sslmode=verify-full&sslrootcert=/run/tls/ca.crt" +$outboxDb = "postgresql+psycopg://$($vars.OUTBOX_DB_USER):$($vars.OUTBOX_DB_PASSWORD)@tls-proxy:15432/$($vars.POSTGRES_DB)?sslmode=verify-full&sslrootcert=/run/tls/ca.crt" +$workerBroker = "amqps://$($vars.WORKER_RABBITMQ_USER):$($vars.WORKER_RABBITMQ_PASSWORD)@tls-proxy:15672/" +$publisherBroker = "amqps://$($vars.PUBLISHER_RABBITMQ_USER):$($vars.PUBLISHER_RABBITMQ_PASSWORD)@tls-proxy:15672/" +$apiCache = "rediss://messenger-api:$($vars.API_REDIS_PASSWORD)@tls-proxy:16379/0?ssl_cert_reqs=required&ssl_ca_certs=/run/tls/ca.crt&ssl_check_hostname=true" +$workerCache = "rediss://messenger-worker:$($vars.WORKER_REDIS_PASSWORD)@tls-proxy:16379/0?ssl_cert_reqs=required&ssl_ca_certs=/run/tls/ca.crt&ssl_check_hostname=true" +$s3Endpoint = $vars.S3_ENDPOINT +$apiSecrets = @{ + DATABASE_URL=$apiDb; CACHE_URL=$apiCache; S3_ENDPOINT=$s3Endpoint + S3_ACCESS_KEY=$vars.S3_ACCESS_KEY_ID; S3_SECRET_KEY=$vars.S3_SECRET_ACCESS_KEY; S3_PUBLIC_ENDPOINT_URL=$vars.S3_PUBLIC_ENDPOINT_URL + OTEL_EXPORTER_OTLP_ENDPOINT=$vars.OTEL_EXPORTER_OTLP_ENDPOINT +} +$workerSecrets = $apiSecrets.Clone() +$workerSecrets.DATABASE_URL = $workerDb +$workerSecrets.CACHE_URL = $workerCache +$workerSecrets.RABBITMQ_URL = $workerBroker +$outboxSecrets = @{ + DATABASE_URL=$outboxDb + RABBITMQ_URL=$publisherBroker +} +BaoPost "secret/data/messenger/api" @{data=$apiSecrets} | Out-Null +BaoPost "secret/data/messenger/worker" @{data=$workerSecrets} | Out-Null +BaoPost "secret/data/messenger/outbox" @{data=$outboxSecrets} | Out-Null + +foreach ($role in @( + @{name="api"; policy="api-policy"}, + @{name="worker"; policy="worker-policy"}, + @{name="outbox"; policy="outbox-policy"} +)) { + BaoPost "auth/approle/role/$($role.name)" @{ + token_policies=@($role.policy); token_ttl="15m"; token_max_ttl="1h" + secret_id_ttl="24h"; secret_id_num_uses=0 + } | Out-Null + $roleId = (Invoke-RestMethod -Method Get -Uri "$base/auth/approle/role/$($role.name)/role-id" -Headers $headers).data.role_id + $secretId = (BaoPost "auth/approle/role/$($role.name)/secret-id" @{}).data.secret_id + $dir = Join-Path $BaoDir $role.name + New-Item -ItemType Directory -Force -Path $dir | Out-Null + Set-Content -NoNewline -Encoding ascii (Join-Path $dir "role_id") $roleId + Set-Content -NoNewline -Encoding ascii (Join-Path $dir "secret_id") $secretId +} +# Restrict the Outbox DB role to its table after migrations have created it. +& docker compose exec -T postgres psql -U $vars.POSTGRES_USER -d $vars.POSTGRES_DB -c "GRANT SELECT, UPDATE ON TABLE outbox_events TO $($vars.OUTBOX_DB_USER)" | Out-Null +if ($LASTEXITCODE -ne 0) { throw "Could not grant Outbox table permissions; run migrations before OpenBao bootstrap." } + +# Distinct Redis ACL identities for API and Worker; the default admin password stays in the Redis container/operator .env only. +& docker compose exec -T redis redis-cli -a $vars.REDIS_ADMIN_PASSWORD ACL SETUSER messenger-api on ">$($vars.API_REDIS_PASSWORD)" "~*" "+@read" "+@write" "+@connection" "+@transaction" "+@scripting" *> $null +if ($LASTEXITCODE -ne 0) { throw "Could not create Redis API ACL user." } +& docker compose exec -T redis redis-cli -a $vars.REDIS_ADMIN_PASSWORD ACL SETUSER messenger-worker on ">$($vars.WORKER_REDIS_PASSWORD)" "~*" "+@read" "+@write" "+@connection" "+@transaction" "+@scripting" *> $null +if ($LASTEXITCODE -ne 0) { throw "Could not create Redis Worker ACL user." } + +& docker compose exec -T redis redis-cli -a $vars.REDIS_ADMIN_PASSWORD --no-auth-warning ACL SAVE *> $null +if ($LASTEXITCODE -ne 0) { throw "Could not save Redis ACL file." } + +function Set-RabbitUser([string]$User,[string]$Password,[string]$Configure,[string]$Write,[string]$Read) { + & docker compose exec -T rabbitmq rabbitmqctl add_user $User $Password *> $null + if ($LASTEXITCODE -ne 0) { + & docker compose exec -T rabbitmq rabbitmqctl change_password $User $Password *> $null + if ($LASTEXITCODE -ne 0) { throw "Could not create/update RabbitMQ service user." } + } + & docker compose exec -T rabbitmq rabbitmqctl set_permissions -p / $User $Configure $Write $Read *> $null + if ($LASTEXITCODE -ne 0) { throw "Could not set RabbitMQ permissions." } +} +Set-RabbitUser $vars.PUBLISHER_RABBITMQ_USER $vars.PUBLISHER_RABBITMQ_PASSWORD '^(attachments|attachment\.process\.v1)$' '^attachments$' '^$' +Set-RabbitUser $vars.WORKER_RABBITMQ_USER $vars.WORKER_RABBITMQ_PASSWORD '^(attachments|attachments\.retry|attachments\.dead|attachment\.process\.v1|attachment\.process\.v1\.retry|attachment\.process\.v1\.dead)$' '^(attachments\.retry|attachments\.dead)$' '^attachment\.process\.v1$' + +$rootToken = $null; $headers = $null +Write-Host "OpenBao policies, AppRoles and scoped RabbitMQ identities configured. Credentials are stored under ignored secrets/bao/." diff --git a/scripts/check-argon2.py b/scripts/check-argon2.py new file mode 100644 index 0000000..799faec --- /dev/null +++ b/scripts/check-argon2.py @@ -0,0 +1,7 @@ +"""Verify the configured Argon2id cost parameters.""" +from app.services.user_service import password_hasher + +sample = password_hasher.hash("Lab7-test-password-only") +assert sample.startswith("$argon2id$v=19$m=19456,t=2,p=1$"), sample +assert password_hasher.verify("Lab7-test-password-only", sample) +print("PASS: Argon2id m=19456 KiB, t=2, p=1; salt is embedded in PHC hash") diff --git a/scripts/check_contract.py b/scripts/check_contract.py index dcb34df..659dd07 100644 --- a/scripts/check_contract.py +++ b/scripts/check_contract.py @@ -9,7 +9,7 @@ ROOT = Path(__file__).resolve().parents[1] def main(): lab = json.loads((ROOT/'lab.json').read_text())['number'] - spec = json.loads((ROOT/'contracts/openapi.json').read_text()) + spec = json.loads((ROOT / 'contracts/openapi.json').read_text(encoding='utf-8')) assert spec['openapi']=='3.1.0' assert spec['info']['version']=='1.%d.0'%lab operation_ids = set() diff --git a/scripts/mtls_probe.py b/scripts/mtls_probe.py new file mode 100644 index 0000000..b6a49e5 --- /dev/null +++ b/scripts/mtls_probe.py @@ -0,0 +1,53 @@ +"""Internal mTLS negative/positive probes; run inside the API container.""" +import argparse +import http.client +import socket +import ssl +import sys + + +def probe(mode: str) -> int: + context = ssl.create_default_context(cafile="/run/tls/ca.crt") + headers = {"Host": "api", "X-Lab7-Client-Identity": "spiffe://lab7/ingress/nginx"} + if mode == "positive": + context.load_cert_chain("/run/tls/lb-client.crt", "/run/tls/lb-client.key") + elif mode == "wrong-identity": + context.load_cert_chain("/run/tls/lb-client.crt", "/run/tls/lb-client.key") + headers["X-Lab7-Client-Identity"] = "spiffe://lab7/worker" + elif mode == "foreign-ca": + context.load_cert_chain("/tmp/foreign-client.crt", "/tmp/foreign-client.key") + elif mode != "no-cert": + raise ValueError(mode) + + try: + raw = socket.create_connection(("127.0.0.1", 8443), timeout=4) + tls = context.wrap_socket(raw, server_hostname="api") + request = "GET /health/live HTTP/1.1\r\n" + request += "".join(f"{k}: {v}\r\n" for k, v in headers.items()) + request += "Connection: close\r\n\r\n" + tls.sendall(request.encode()) + response = tls.recv(4096).decode("latin1", errors="replace") + tls.close() + status = int(response.split(" ", 2)[1]) + except (ssl.SSLError, OSError) as exc: + if mode in {"no-cert", "foreign-ca"}: + print(f"PASS: {mode} rejected during TLS handshake ({type(exc).__name__})") + return 0 + raise + + if mode == "positive" and status == 200: + print("PASS: allowed ingress identity accepted") + return 0 + if mode == "wrong-identity" and status == 403: + print("PASS: non-ingress identity rejected by API authorization") + return 0 + if mode in {"no-cert", "foreign-ca"}: + raise AssertionError(f"{mode} unexpectedly reached HTTP status {status}") + raise AssertionError(f"{mode} expected its security outcome, got HTTP {status}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=["no-cert", "foreign-ca", "wrong-identity", "positive"]) + args = parser.parse_args() + sys.exit(probe(args.mode)) diff --git a/scripts/replay_failed_job.py b/scripts/replay_failed_job.py new file mode 100644 index 0000000..3c32904 --- /dev/null +++ b/scripts/replay_failed_job.py @@ -0,0 +1,158 @@ +"""Safely replay a failed attachment processing job. + +Usage: + python scripts/replay_failed_job.py ATTACHMENT_UUID +""" + +import argparse +import logging +import sys +import uuid +from datetime import datetime, timezone +from pathlib import Path + +# Add project root to Python import path. +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT)) + +from sqlalchemy import select + +from app.db.models import Attachment, OutboxEvent, ProcessingJob +from app.db.session import SessionLocal + + +logger = logging.getLogger("replay_failed_job") + +EVENT_TYPE = "attachment.process.v1" + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def replay_failed_job(attachment_id: uuid.UUID) -> None: + db = SessionLocal() + + try: + # Lock the attachment first to serialize concurrent replay requests. + attachment = db.scalar( + select(Attachment) + .where(Attachment.id == attachment_id) + .with_for_update() + ) + + if attachment is None: + raise ValueError(f"Attachment not found: {attachment_id}") + + job = db.scalar( + select(ProcessingJob) + .where( + ProcessingJob.attachment_id == attachment_id, + ProcessingJob.pipeline_version == attachment.pipeline_version, + ) + .with_for_update() + ) + + if job is None: + raise ValueError( + f"Processing job not found for attachment: {attachment_id}" + ) + + # Only terminally failed jobs may be replayed. + if job.status != "failed": + raise ValueError( + f"Job cannot be replayed: current job status is '{job.status}'" + ) + + if attachment.status != "failed": + raise ValueError( + "Attachment and job states are inconsistent: " + f"attachment status is '{attachment.status}'" + ) + + # The worker needs the original staged file. + if not attachment.staging_key: + raise ValueError( + "Cannot replay: staging_key is empty. " + "The original file is not available for processing." + ) + + now = utc_now() + + # Reset the existing job. Keep its ID and pipeline version. + job.attempt = 0 + job.status = "queued" + job.available_at = now + job.lease_until = None + job.worker_id = None + job.last_error_code = None + + # Return attachment metadata to the queued state. + attachment.status = "queued" + attachment.error_code = None + + # Create a new durable outbox event in the same transaction. + event = OutboxEvent( + event_type=EVENT_TYPE, + aggregate_id=attachment.id, + payload={ + "schema_version": 1, + "job_id": str(job.id), + "type": EVENT_TYPE, + "attachment_id": str(attachment.id), + "pipeline_version": job.pipeline_version, + "attempt": 1, + "created_at": now.isoformat(), + }, + status="pending", + ) + + db.add(event) + db.commit() + + print("Replay request successfully committed.") + print(f"Attachment ID: {attachment.id}") + print(f"Job ID: {job.id}") + print(f"Job status: {job.status}") + print(f"Attempt: {job.attempt}") + print(f"Outbox event: {event.id}") + print("Outbox status: pending") + print() + print("The existing outbox-publisher will publish the event.") + + except Exception: + db.rollback() + raise + + finally: + db.close() + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Replay a terminally failed attachment processing job." + ) + parser.add_argument( + "attachment_id", + help="UUID of the attachment to replay", + ) + + args = parser.parse_args() + + try: + attachment_id = uuid.UUID(args.attachment_id) + replay_failed_job(attachment_id) + return 0 + + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + except Exception: + logger.exception("Could not replay failed processing job") + return 1 + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + raise SystemExit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..4137811 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu + +# Restart persistent services without deleting their volumes. +docker compose restart postgres redis seaweedfs + +# Restart RabbitMQ first, then refresh stunnel DNS resolution. +docker compose restart rabbitmq +docker compose restart tls-proxy + +# Restart application services after their dependencies. +docker compose restart api worker outbox-publisher diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..fe4c594 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test must provide the API instance id}" +found="" +for container in $(docker compose ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container") + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + found="$container" + break + fi +done +if [ -z "$found" ]; then + echo "No API container matched TARGET_INSTANCE_ID" >&2 + exit 1 +fi +docker stop "$found" >/dev/null diff --git a/scripts/test-login-limit.py b/scripts/test-login-limit.py new file mode 100644 index 0000000..872233b --- /dev/null +++ b/scripts/test-login-limit.py @@ -0,0 +1,40 @@ +"""Exercise Nginx login rate limiting; run against the HTTPS ingress.""" +import argparse +import base64 +import ssl +import time +import urllib.error +import urllib.request + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default="https://localhost:8443") + p.add_argument("--ca-file", default="secrets/tls/ca.crt") + args = p.parse_args() + context = ssl.create_default_context(cafile=args.ca_file) + auth = base64.b64encode(b"not-a-user:deliberately-wrong-password").decode() + statuses = [] + retry_after = None + for _ in range(12): + request = urllib.request.Request( + args.base_url.rstrip("/") + "/api/v1/auth/sessions", + data=b"{}", + headers={"Authorization": "Basic " + auth, "Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, context=context, timeout=5) as response: + statuses.append(response.status) + except urllib.error.HTTPError as exc: + statuses.append(exc.code) + if exc.code == 429: + retry_after = exc.headers.get("Retry-After") + time.sleep(0.05) + assert 429 in statuses, f"rate limiter did not return 429; statuses={statuses}" + assert retry_after, "429 response lacks Retry-After" + print("PASS: login rate limit returned 429 with Retry-After") + + +if __name__ == "__main__": + main() diff --git a/scripts/test-mtls.ps1 b/scripts/test-mtls.ps1 new file mode 100644 index 0000000..5635269 --- /dev/null +++ b/scripts/test-mtls.ps1 @@ -0,0 +1,21 @@ +$ErrorActionPreference = "Stop" +$Root = Split-Path -Parent $PSScriptRoot +$Tls = Join-Path $Root "secrets/tls" +if (-not (Get-Command openssl -ErrorAction SilentlyContinue)) { throw "OpenSSL is required." } +function Run-OpenSSL([string[]]$Args) { + & openssl @Args + if ($LASTEXITCODE -ne 0) { throw "OpenSSL failed: $Args" } +} +$foreign="$Tls/foreign" +New-Item -ItemType Directory -Force -Path $foreign | Out-Null +Run-OpenSSL @("req","-x509","-newkey","rsa:2048","-nodes","-keyout","$foreign/ca.key","-out","$foreign/ca.crt","-days","30","-subj","/CN=Foreign Test CA") +Run-OpenSSL @("req","-newkey","rsa:2048","-nodes","-keyout","$foreign/client.key","-out","$foreign/client.csr","-subj","/CN=foreign-client") +Run-OpenSSL @("x509","-req","-in","$foreign/client.csr","-CA","$foreign/ca.crt","-CAkey","$foreign/ca.key","-CAcreateserial","-out","$foreign/client.crt","-days","30") +$container = (docker compose ps -q api | Select-Object -First 1) +if (-not $container) { throw "API container is not running." } +docker cp "$foreign/client.crt" "${container}:/tmp/foreign-client.crt" +docker cp "$foreign/client.key" "${container}:/tmp/foreign-client.key" +docker compose exec api python scripts/mtls_probe.py no-cert +docker compose exec api python scripts/mtls_probe.py foreign-ca +docker compose exec api python scripts/mtls_probe.py wrong-identity +docker compose exec api python scripts/mtls_probe.py positive diff --git a/scripts/test-openbao-policy.py b/scripts/test-openbao-policy.py new file mode 100644 index 0000000..0ab1008 --- /dev/null +++ b/scripts/test-openbao-policy.py @@ -0,0 +1,47 @@ +"""Check that AppRole tokens are scoped to their own KV v2 path.""" +import json +from pathlib import Path +import requests + +ROOT = Path(__file__).resolve().parents[1] +BASE = "https://localhost:18200/v1" +CA = ROOT / "secrets" / "tls" / "ca.crt" + + +def token_for(role: str) -> str: + role_dir = ROOT / "secrets" / "bao" / role + response = requests.post( + BASE + "/auth/approle/login", + json={ + "role_id": (role_dir / "role_id").read_text().strip(), + "secret_id": (role_dir / "secret_id").read_text().strip(), + }, + verify=str(CA), + timeout=5, + ) + response.raise_for_status() + return response.json()["auth"]["client_token"] + + +def check_role(role: str, allowed: str, denied: str) -> None: + token = token_for(role) + headers = {"X-Vault-Token": token} + own = requests.get(BASE + f"/secret/data/messenger/{allowed}", headers=headers, verify=str(CA), timeout=5) + assert own.status_code == 200, f"{role} cannot read its own path" + other = requests.get(BASE + f"/secret/data/messenger/{denied}", headers=headers, verify=str(CA), timeout=5) + assert other.status_code == 403, f"{role} unexpectedly read {denied}" + write = requests.post( + BASE + f"/secret/data/messenger/{allowed}", + headers=headers, + json={"data": {"forbidden_write": "must-not-be-written"}}, + verify=str(CA), + timeout=5, + ) + assert write.status_code == 403, f"{role} unexpectedly wrote its own path" + print(f"PASS: {role} reads {allowed}; denied {denied} and write") + + +if __name__ == "__main__": + check_role("api", "api", "worker") + check_role("worker", "worker", "api") + check_role("outbox", "outbox", "api") diff --git a/scripts/test-session-expiry.py b/scripts/test-session-expiry.py new file mode 100644 index 0000000..2db3598 --- /dev/null +++ b/scripts/test-session-expiry.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Verify that an expired session is rejected through HTTPS.""" + +import hashlib +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, arguments, run + + +def database_query(sql): + result = subprocess.run( + [ + "docker", "compose", "exec", "-T", + "-e", "SQL=" + sql, + "postgres", + "sh", "-lc", + 'psql -X -A -t -v ON_ERROR_STOP=1 ' + '-U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$SQL"', + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + raise Failure("PostgreSQL operation failed") + + return result.stdout.strip() + + +def main(): + args = arguments("Expired session security test") + client = Client(args.base_url, args.ca_file) + + actor = client.user("expiry") + token = actor["token"] + token_hash = hashlib.sha256(token.encode("utf-8")).hexdigest() + + try: + updated = database_query( + "UPDATE sessions " + "SET expires_at = NOW() - INTERVAL '1 second' " + f"WHERE token_hash = '{token_hash}' " + "RETURNING id;" + ) + + if not updated: + raise Failure("Test session was not found in PostgreSQL") + + client.call( + "GET", + "/api/v1/users/me", + 401, + actor=actor, + ) + + print("PASS: expired session rejected with HTTP 401") + + finally: + database_query( + f"DELETE FROM sessions WHERE token_hash = '{token_hash}';" + ) + + +if __name__ == "__main__": + run(main) diff --git a/tests/client.py b/tests/client.py index f0499a2..4dd6c88 100644 --- a/tests/client.py +++ b/tests/client.py @@ -1,4 +1,4 @@ -"""HTTP helpers for public black-box tests; Python 3.9+, standard library only.""" +"""HTTP helpers for public black-box tests; Python 3.9+, standard library only.""" import base64 import datetime as dt import hashlib @@ -14,7 +14,7 @@ import uuid ROOT = Path(__file__).resolve().parents[1] LAB = json.loads((ROOT / 'lab.json').read_text())['number'] -SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text()) +SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text(encoding='utf-8-sig')) class Failure(AssertionError): pass @@ -215,3 +215,4 @@ def run(main): except (Failure,ValueError,OSError) as exc: print('FAIL:', str(exc)) raise SystemExit(1) + diff --git a/tests/resilience_windows.py b/tests/resilience_windows.py new file mode 100644 index 0000000..2a07e95 --- /dev/null +++ b/tests/resilience_windows.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Explicit operator-authored hook; tokens remain in memory, volumes are never removed by this test.""" +import json +import os +from pathlib import Path +import subprocess +import time +import uuid +from urllib.parse import quote +from client import Client, Failure, LAB, arguments, check, run + + +def history(c, actor, chat): + path = '/api/v1/chats/'+chat['id']+'/messages?limit=100' + result = [] + seen = set() + for _ in range(1000): + page = c.call('GET',path,actor=actor,schema='MessagePage')['json'] + result.extend(page['items']) + cursor = page['next_cursor'] + if cursor is None: + return result + check(cursor not in seen,'cursor cycle') + seen.add(cursor) + path = '/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+quote(cursor,safe='') + raise Failure('history pagination exceeded safety bound') + + +def main(): + def configure(p): + p.add_argument('--mode', choices=['persistence','failover'], required=True) + p.add_argument('--action-script',required=True,help='Executable script authored by student; restarts stack or stops ONE API replica') + args = arguments('Explicit restart/failover scenario; modifies only synthetic API data',configure) + check(LAB>=2,'persistence requires lab2+') + check(args.mode!='failover' or LAB>=3,'failover requires lab3+') + script = Path(args.action_script).expanduser().resolve() + check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)') + check('.example.' not in script.name,'copy and implement the example hook first') + c = Client(args.base_url,args.ca_file,timeout=2) + alice,bob,eve,chat = c.bootstrap() + sent = c.message(alice,chat,'Сохранить при отказе') + original = sent['json'] + target = sent['headers'].get('X-Instance-Id','') + attachment = content = None + if LAB>=4: + attachment,content = c.attachment(alice,chat) + if LAB>=5: + attachment = c.wait_ready(alice,attachment) + env = dict(os.environ) + env['TARGET_INSTANCE_ID'] = target + observed = set() + if args.mode=='failover': + for _ in range(40): + observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id']) + check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity') + print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.') + # No shell interpolation and no Docker commands in this test. + proc = subprocess.Popen([r"C:\Program Files\Git\bin\bash.exe", str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + started = time.monotonic() + successes,failures,after_instances = [],0,[] + try: + if args.mode=='persistence': + try: + code = proc.wait(timeout=90) + except subprocess.TimeoutExpired: + raise Failure('restart hook exceeded 90 seconds') from None + check(code==0,'restart hook failed (inspect your script locally)') + deadline = time.monotonic()+90 + while True: + try: + c.call('GET','/health/ready',schema='Health') + c.call('GET','/api/v1/users/me',actor=alice,schema='User') + break + except Failure: + check(time.monotonic()=5,'too few successful writes after stop') + check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped') + print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False)) + # Existing token, no re-login. Read all confirmations and reject duplicates. + messages = history(c,bob,chat) + ids = [m['id'] for m in messages] + check(len(ids)==len(set(ids)),'duplicated message IDs in history') + check(any(m==original for m in messages),'confirmed pre-failure message lost or altered') + for msg in successes: + check(any(m==msg for m in messages),'confirmed in-flight message lost or altered') + # Bodies are unique in this scenario: an uncertain request may exist once, never twice. + texts = [m['text'] for m in messages] + check(len(texts)==len(set(texts)),'uncertain retry created duplicate message') + if attachment: + c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment') + downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob) + check(downloaded['bytes']==content,'attachment lost or changed across failure') + print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.') + finally: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + +if __name__=='__main__': + run(main) + diff --git a/tests/test_login_rate_limit.py b/tests/test_login_rate_limit.py new file mode 100644 index 0000000..dd52965 --- /dev/null +++ b/tests/test_login_rate_limit.py @@ -0,0 +1,55 @@ + +#!/usr/bin/env python3 +import base64 +import uuid + +from client import Client, arguments, check, run + + +def main(): + args = arguments("Login rate limit security test") + client = Client(args.base_url, args.ca_file) + + username = "rate_" + uuid.uuid4().hex[:12] + password = "Correct-" + uuid.uuid4().hex + + client.call( + "POST", + "/api/v1/users", + 201, + body={ + "username": username, + "display_name": "Rate limit test", + "password": password, + }, + schema="User", + ) + + wrong = base64.b64encode( + f"{username}:Wrong-{uuid.uuid4().hex}".encode() + ).decode() + + for attempt in range(5): + client.call( + "POST", + "/api/v1/auth/sessions", + 401, + headers={"Authorization": "Basic " + wrong}, + ) + + blocked = client.call( + "POST", + "/api/v1/auth/sessions", + 429, + headers={"Authorization": "Basic " + wrong}, + ) + + retry_after = blocked["headers"].get("Retry-After") + check(retry_after is not None, "429 response has no Retry-After") + check(int(retry_after) > 0, "Retry-After must be positive") + + print("PASS: login rate limit returns 429 and Retry-After") + + +if __name__ == "__main__": + run(main)