diff --git a/.dockerignore b/.dockerignore index bc16e556..9b42bc44 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,6 +14,7 @@ backups *.pfx *.jks .venv +.tools __pycache__ node_modules evidence diff --git a/.env.example b/.env.example index d99d26a2..bfaff572 100644 --- a/.env.example +++ b/.env.example @@ -4,17 +4,20 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +# Values below are local synthetic-stack configuration. Replace placeholders in .env. +POSTGRES_USER=messenger +POSTGRES_PASSWORD=replace-db-password-with-random-value +POSTGRES_DB=messenger +MINIO_ROOT_USER=minio-admin +MINIO_ROOT_PASSWORD=replace-minio-root-password-with-random-value +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY=messenger-api +S3_SECRET_KEY=replace-s3-secret-with-random-value +MINIO_ROOT_USER=minio-admin +MINIO_ROOT_PASSWORD=replace-with-a-random-32-character-value +S3_ACCESS_KEY=messenger-api +S3_SECRET_KEY=replace-with-a-different-random-32-character-value MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..9082c7a6 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Зюлина Екатерина Михайловна / БВТ2302 +Целевая оценка (3/4/5): 3 (только обязательный работающий минимум) +Трек 5, если заявлен: не заявлен -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: приложение и миграции lab3, attachment metadata в PostgreSQL, приватные bytes в MinIO, raw upload/download PNG/JPEG, привязка вложений к сообщениям и chat ACL, обязательные размерные/content проверки, Compose с двумя API репликами. +Команды запуска из чистого clone: раздел «Запуск из чистого clone» в REPORT.md. +Результаты проверок и ссылка на REPORT.md: contract, Compose, OpenAPI, smoke, persistence/failover и S3 outage прошли; см. таблицу и evidence в REPORT.md. +Ограничения / известные ошибки: Docker-host, PostgreSQL, Redis, MinIO и LB остаются SPOF; автоматическая сверка orphan объектов не входит в целевой уровень. +Источники и использование ИИ: курс/контракт, собственная lab3 база; Codex применялся для анализа и реализации. Сторонние исходники не переносились. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Обязательные возможности предыдущих лабораторных сохранены с переходом `attachment_ids=[]`. +- [x] Контракт и публичные тесты не ослаблены. +- [x] `make check` и `make test`-эквивалентные команды выполнены; реальный вывод приложен. +- [x] Сценарии persistence/failover и S3 недоступности уровня 3 воспроизведены. +- [x] Секреты локальны и игнорируются Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию решения. diff --git a/.gitignore b/.gitignore index b5a71896..05e279e3 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ data/ backups/ # Runtime/build artifacts .venv/ +.tools/ __pycache__/ *.py[cod] node_modules/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..997811e7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.11.10-slim-bookworm + +WORKDIR /app +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 + +# Устанавливаем зависимости +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Копируем код приложения +COPY src/ ./src/ +COPY alembic/ ./alembic/ +COPY alembic.ini . +COPY entrypoint.sh . + +RUN chmod 755 entrypoint.sh && groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app + +USER 10001:10001 + +# Открываем порт +EXPOSE 8080 + +ENTRYPOINT ["./entrypoint.sh"] + +# Команда запуска +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--limit-concurrency", "64", "--timeout-keep-alive", "5"] diff --git a/Makefile b/Makefile index 5d86d171..b50ad998 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-s3-outage help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -28,3 +28,6 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-s3-outage: + $(PYTHON) student-tests/s3_unavailable.py --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..51bbff7f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,100 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Зюлина Екатерина Михайловна, БВТ2302. +- Целевая оценка: 3. +- Commit/PR: заполнить после публикации. +- Источники, библиотеки и помощь ИИ: контракт и материалы текущего skeleton; собственная реализация lab3; FastAPI, SQLAlchemy, Alembic, boto3, Pillow; [MinIO image mirror](https://hub.docker.com/r/dappros/minio) как закреплённый S3 server образ; Codex использовался для анализа требований, переноса и реализации. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Для Windows PowerShell и Docker Desktop с Linux containers нужен Docker Compose v2 и Python 3.11 на хосте для запуска публичных тестов. Runtime в Docker закреплён на Python 3.11.10; API-зависимости перечислены с версиями в `requirements.txt`. Секреты генерируются локально в игнорируемом `.env`; их значения не хранятся в репозитории. В PowerShell из чистого clone выполните: + +```powershell +$dbSecret = [Convert]::ToHexString([Security.Cryptography.RandomNumberGenerator]::GetBytes(24)) +$rootSecret = [Convert]::ToHexString([Security.Cryptography.RandomNumberGenerator]::GetBytes(24)) +$s3Secret = [Convert]::ToHexString([Security.Cryptography.RandomNumberGenerator]::GetBytes(32)) +$envText = Get-Content -Raw .env.example +$envText = $envText.Replace('replace-db-password-with-random-value', $dbSecret) +$envText = $envText.Replace('replace-minio-root-password-with-random-value', $rootSecret) +$envText = $envText.Replace('replace-s3-secret-with-random-value', $s3Secret) +[IO.File]::WriteAllText((Join-Path (Get-Location) '.env'), $envText, [Text.UTF8Encoding]::new($false)) +$env:PYTHONUTF8 = '1' +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +do { Start-Sleep -Seconds 2; try { Invoke-WebRequest http://localhost:8080/health/ready -UseBasicParsing | Out-Null; $ready = $true } catch { $ready = $false } } until ($ready) +py -3.11 scripts/check_contract.py +py -3.11 tests/smoke.py --base-url http://localhost:8080 +py -3.11 tests/resilience.py --mode persistence --action-script scripts/restart.bat --base-url http://localhost:8080 +py -3.11 tests/resilience.py --mode failover --action-script scripts/stop-one.bat --base-url http://localhost:8080 +docker compose --env-file .env start api1 api2 +py -3.11 student-tests/s3_unavailable.py --base-url http://localhost:8080 +``` + +Тесты создают пользователей и чаты, которые не удаляются API. Повторяйте проверки на отдельном учебном volume. Make-цели `make check` и `make test` соответствуют Python-командам выше. + +Образы закреплены версиями: `postgres:16.4-alpine`, `redis:7.4.2-alpine`, `dappros/minio:RELEASE.2025-09-07T16-13-09Z` (зеркальная копия MinIO RELEASE), `ghcr.io/golithus/mc:RELEASE.2025-08-13T08-35-41Z` (зафиксирован digest `sha256:dde55ccc0bc9e65dc582c8e652d652eea1438c6e60d774524f4da15d7b26a900`), `nginx:1.27.5-alpine`. API образ собирается из Dockerfile текущего репозитория как `mtusi-messenger-api:4.0.0`. + +Схема БД создаётся одноразовым сервисом `migrate` командой `alembic upgrade head`. Сервис `s3-bootstrap` настраивает закрытый от anonymous bucket. DB, Redis и MinIO сохраняются в named volumes. Из API наружу опубликован только LB на `localhost:8080`. + +### Сохранность и отказ + +```powershell +py -3.11 tests/resilience.py --mode persistence --action-script scripts/restart.bat --base-url http://localhost:8080 +py -3.11 tests/resilience.py --mode failover --action-script scripts/stop-one.bat --base-url http://localhost:8080 +docker compose --env-file .env start api1 api2 +py -3.11 student-tests/s3_unavailable.py --base-url http://localhost:8080 +``` + +Failover последовательно останавливает обе API-реплики. После сценария запустите их обе и дождитесь `/health/ready`. Restart hook перезапускает PostgreSQL, Redis, MinIO, API и LB с сохранением named volumes; данные и volumes не удаляются. Тест отказа останавливает MinIO, проверяет недоступность readiness/upload и затем восстанавливает сервис. + +Для Linux перед запуском hooks задайте executable bit: `chmod +x scripts/restart.sh scripts/stop-one.sh`. При подготовке Git-коммита из Windows зафиксируйте режим файлов командой `git update-index --chmod=+x scripts/restart.sh scripts/stop-one.sh` после их добавления в index. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Путь запроса: клиент → Nginx → две API-реплики → общие PostgreSQL, Redis и приватный MinIO. Данные пользователей, сессии, чаты, сообщения, идемпотентность и метаданные вложений хранятся в PostgreSQL; байты оригинала — только в S3. Ключ `original/` генерирует сервер. Исходное имя хранится как display filename и не используется в S3 key. + +Каждая реплика ограничивает HTTP body 10 MiB. Pillow проверяет реальный формат, целостность и размеры, отклоняя изображения больше 40 млн пикселей. Проверяются MIME, filename, chat membership и UUID. Ответ `201 ready` выдаётся после сохранения объекта в S3 и metadata в PostgreSQL. GET metadata/content повторно проверяют членство в чате; API не отдаёт redirect или S3 key. + +Порядок upload: сначала S3 PUT, затем metadata в PostgreSQL; ответ `201 ready` возвращается после обоих операций. При отказе S3 API возвращает JSON 503. При сбое БД после успешного PUT может остаться объект без metadata; автоматическая сверка и очистка не входят в целевой уровень этой работы. + +Из lab3 перенесены Python API, PostgreSQL/Redis store, сессионная аутентификация, межрепличная идемпотентность, Alembic migrations, Dockerfile, Nginx и restart/failover hooks. Добавлены миграция `004_attachments`, S3 слой, binary endpoints, attachment references в сообщениях, MinIO/bootstrap. Изменения контракта ограничены переходом lab4: старые текстовые сообщения сохраняют `attachment_ids=[]`; upload возвращает 201 ready. + +| Роль | Service / версия | Health/readiness | Volume, сеть и доступ | +| --- | --- | --- | --- | +| LB | `lb`, nginx:1.27.5-alpine | Compose process; активный probe API — `health/ready` | Host `localhost:8080`; client body max 11 MiB для JSON 413 приложения на 10 MiB; не ретраит POST | +| API | `api1`, `api2`, локальный Python 3.11.10 image | `/health/ready` проверяет PostgreSQL, Redis и bucket | Только внутренняя Compose-сеть; S3 доступ через credentials из локального `.env` | +| Миграции | `migrate`, локальный API image | `alembic upgrade head`, однократный exit 0 | PostgreSQL; до API | +| PostgreSQL | `db`, postgres:16.4-alpine | SQL readiness проверяется API | `postgres_data`; сеть внутренняя | +| Session cache | `redis`, redis:7.4.2-alpine | Redis ping проверяется API | `redis_data`, AOF `appendfsync always`; внутренняя сеть | +| Binary store | `minio`, dappros mirror of MinIO RELEASE.2025-09-07T16-13-09Z | `/minio/health/ready`, bucket probe в API | `minio_data`; S3 API только внутри сети, console не опубликована | +| Bucket bootstrap | `s3-bootstrap`, mc RELEASE.2025-08-13T08-35-41Z, GHCR digest закреплён | Одноразовый успешный exit | Изменяет bucket/policy/user через локальные `.env` credentials | ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Проверки проведены на Compose стенде 12.11.2026. Contract, Compose config/build, OpenAPI/schema validation, публичный smoke, persistence, failover и S3 outage прошли. Операторские записи hooks — в [`evidence/persistence-actions.txt`](evidence/persistence-actions.txt) и [`evidence/failover-actions.txt`](evidence/failover-actions.txt). После проверок все сервисы и обе API реплики healthy. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда/сценарий | Результат и доказательство | +| --- | --- | --- | +| Структура skeleton и контракт | `scripts/check_contract.py` | PASS; обязательные skeleton-файлы сохранены. | +| Синтаксис реализации | Offline Python compile | PASS. | +| Compose configuration и сборка | `docker compose config --quiet`, `docker compose build` | PASS; стенд запущен. | +| Полная OpenAPI 3.1 и JSON Schema | `scripts/validate_openapi.py` с `requirements-dev.txt` | PASS. | +| Публичный smoke и доступ участников к вложению | `tests/smoke.py --base-url http://127.0.0.1:8080` | PASS; участник скачал оригинал, постороннему отказано. | +| Failover и persistence | `tests/resilience.py --mode failover/persistence` | PASS: 95 успешных записей, 0 ошибок, recovery 3.234 s; persistence сохранил данные и сессию. StartedAt записан в evidence. | +| S3 outage и восстановление | `student-tests/s3_unavailable.py` | PASS: contract JSON 503/504 для readiness/upload и восстановление после старта MinIO. | -## Эксперименты и отказы +Evidence содержит сводные метрики, состояния сервисов до/после hooks и идентификаторы тестовых target без секретов. Не добавляйте `.env`, токены или реальные данные. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Проверки отказа -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Воздействие | Ожидаемое поведение | Наблюдение / восстановление | +| --- | --- | --- | +| MinIO недоступен при readiness/upload | Public gateway возвращает contract JSON 503 с request ID; upload не становится ready | PASS; MinIO запущен повторно, readiness восстановилась | +| API остановлена после upload | LB переключает запрос на вторую реплику; bytes и metadata доступны | PASS: 95 записей, 0 failed attempts, recovery 3.234 s | +| Restart PostgreSQL/Redis/MinIO/API/LB с volumes | прежняя сессия и attachment bytes остаются доступны | PASS; StartedAt до/после сохранён в persistence evidence | + +Стенд на одном Docker host переживает остановку одной API-реплики, но не потерю host, LB, PostgreSQL, Redis или самого MinIO. MinIO тут single-node; репликация S3 и HA БД не заявлены. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Реализация и отчёт ограничены критериями уровня 3; функции уровней 4/5 не заявляются. Одна реплика MinIO и одна БД являются SPOF. Если PostgreSQL откажет после успешного S3 PUT, объект может остаться без записи metadata; автоматическая сверка не реализована. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..c4365907 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,37 @@ +[alembic] +script_location = alembic +sqlalchemy.url = postgresql://postgres:postgres@db:5432/messenger + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 00000000..b75cbcf2 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,46 @@ +from logging.config import fileConfig +from sqlalchemy import engine_from_config +from alembic import context +import sys +import os + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from src.database import Base, DATABASE_URL + +config = context.config +config.set_main_option("sqlalchemy.url", DATABASE_URL) + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def run_migrations_offline(): + url = context.config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=None, + ) + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() \ No newline at end of file diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 00000000..fe11c3e3 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,22 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} \ No newline at end of file diff --git a/alembic/versions/001_initial.py b/alembic/versions/001_initial.py new file mode 100644 index 00000000..14154290 --- /dev/null +++ b/alembic/versions/001_initial.py @@ -0,0 +1,70 @@ +"""Initial schema + +Revision ID: 001 +Revises: +Create Date: 2026-10-12 +""" +from alembic import op +import sqlalchemy as sa + +revision = '001' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table('users', + sa.Column('id', sa.String(), nullable=False), + sa.Column('username', sa.String(length=32), nullable=False), + sa.Column('display_name', sa.String(length=100), nullable=False), + sa.Column('password_hash', sa.String(length=255), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('id')) + op.create_index('ix_users_username', 'users', ['username'], unique=True) + + op.create_table('sessions', + sa.Column('id', sa.String(), nullable=False), + sa.Column('token', sa.String(length=255), nullable=False), + sa.Column('user_id', sa.String(), nullable=False), + sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id']), + sa.PrimaryKeyConstraint('id')) + op.create_index('ix_sessions_token', 'sessions', ['token'], unique=True) + + op.create_table('chats', + sa.Column('id', sa.String(), nullable=False), + sa.Column('title', sa.String(length=100), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('id')) + + op.create_table('chat_members', + sa.Column('id', sa.String(), nullable=False), + sa.Column('chat_id', sa.String(), nullable=False), + sa.Column('user_id', sa.String(), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id']), + sa.ForeignKeyConstraint(['user_id'], ['users.id']), + sa.PrimaryKeyConstraint('id')) + + op.create_table('messages', + sa.Column('id', sa.String(), nullable=False), + sa.Column('chat_id', sa.String(), nullable=False), + sa.Column('sender_id', sa.String(), nullable=False), + sa.Column('text', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id']), + sa.ForeignKeyConstraint(['sender_id'], ['users.id']), + sa.PrimaryKeyConstraint('id')) + op.create_index('ix_messages_chat_created', 'messages', ['chat_id', 'created_at']) + + +def downgrade(): + op.drop_index('ix_messages_chat_created', table_name='messages') + op.drop_table('messages') + op.drop_table('chat_members') + op.drop_table('chats') + op.drop_index('ix_sessions_token', table_name='sessions') + op.drop_table('sessions') + op.drop_index('ix_users_username', table_name='users') + op.drop_table('users') diff --git a/alembic/versions/002_history_constraints.py b/alembic/versions/002_history_constraints.py new file mode 100644 index 00000000..8d7699be --- /dev/null +++ b/alembic/versions/002_history_constraints.py @@ -0,0 +1,35 @@ +"""Add membership uniqueness and the keyset history index. + +Revision ID: 002 +Revises: 001 +Create Date: 2026-10-14 +""" +from alembic import op +import sqlalchemy as sa + +revision = '002' +down_revision = '001' +branch_labels = None +depends_on = None + + +def upgrade(): + # Older schemas allowed nullable timestamps. Backfill before tightening the + # constraint so the migration also works on an existing populated database. + for table in ('users', 'sessions', 'chats', 'messages'): + op.execute(sa.text(f'UPDATE {table} SET created_at = CURRENT_TIMESTAMP WHERE created_at IS NULL')) + op.alter_column(table, 'created_at', existing_type=sa.DateTime(timezone=True), nullable=False) + + op.create_unique_constraint('uq_chat_member', 'chat_members', ['chat_id', 'user_id']) + op.create_index('ix_chat_members_user_chat', 'chat_members', ['user_id', 'chat_id']) + op.drop_index('ix_messages_chat_created', table_name='messages') + op.create_index('ix_messages_chat_created_id', 'messages', ['chat_id', 'created_at', 'id']) + + +def downgrade(): + op.drop_index('ix_messages_chat_created_id', table_name='messages') + op.create_index('ix_messages_chat_created', 'messages', ['chat_id', 'created_at']) + op.drop_index('ix_chat_members_user_chat', table_name='chat_members') + op.drop_constraint('uq_chat_member', 'chat_members', type_='unique') + for table in ('users', 'sessions', 'chats', 'messages'): + op.alter_column(table, 'created_at', existing_type=sa.DateTime(timezone=True), nullable=True) diff --git a/alembic/versions/003_message_idempotency.py b/alembic/versions/003_message_idempotency.py new file mode 100644 index 00000000..890acc9c --- /dev/null +++ b/alembic/versions/003_message_idempotency.py @@ -0,0 +1,30 @@ +"""Persist message idempotency claims in the shared PostgreSQL database.""" +from alembic import op +import sqlalchemy as sa + +revision = "003_message_idempotency" +down_revision = "002" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "message_idempotency", + sa.Column("id", sa.String(), nullable=False), + sa.Column("user_id", sa.String(), nullable=False), + sa.Column("method", sa.String(length=8), nullable=False), + sa.Column("chat_path", sa.String(length=80), nullable=False), + sa.Column("key", sa.String(length=128), nullable=False), + sa.Column("request_hash", sa.String(length=64), nullable=False), + sa.Column("message_id", sa.String(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["message_id"], ["messages.id"]), + sa.ForeignKeyConstraint(["user_id"], ["users.id"]), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id", "method", "chat_path", "key", name="uq_idempotency_scope"), + ) + + +def downgrade(): + op.drop_table("message_idempotency") diff --git a/alembic/versions/004_attachments.py b/alembic/versions/004_attachments.py new file mode 100644 index 00000000..5b79c45b --- /dev/null +++ b/alembic/versions/004_attachments.py @@ -0,0 +1,38 @@ +"""Add attachment metadata and message references. + +Revision ID: 004_attachments +Revises: 003_message_idempotency +""" +from alembic import op +import sqlalchemy as sa + +revision = "004_attachments" +down_revision = "003_message_idempotency" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column("messages", sa.Column("attachment_ids", sa.JSON(), nullable=False, + server_default=sa.text("'[]'"))) + op.create_table( + "attachments", + sa.Column("id", sa.String(), primary_key=True), + sa.Column("chat_id", sa.String(), sa.ForeignKey("chats.id"), nullable=False), + sa.Column("owner_id", sa.String(), sa.ForeignKey("users.id"), nullable=False), + sa.Column("filename", sa.String(255), nullable=False), + sa.Column("content_type", sa.String(32), nullable=False), + sa.Column("size_bytes", sa.Integer(), nullable=False), + sa.Column("sha256", sa.String(64), nullable=False), + sa.Column("object_key", sa.String(128), nullable=False, unique=True), + sa.Column("status", sa.String(16), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index("ix_attachments_chat_id", "attachments", ["chat_id"]) + + +def downgrade(): + op.drop_index("ix_attachments_chat_id", table_name="attachments") + op.drop_table("attachments") + op.drop_column("messages", "attachment_ids") diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..cff06245 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,123 @@ +services: + api1: + build: . + image: mtusi-messenger-api:4.0.0 + environment: &api-env + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?Create a local .env first}@db:5432/${POSTGRES_DB:-messenger} + CACHE_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Create local S3 credentials} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Create local S3 credentials} + depends_on: + migrate: {condition: service_completed_successfully} + s3-bootstrap: {condition: service_completed_successfully} + db: {condition: service_started} + redis: {condition: service_started} + mem_limit: 512m + cpus: 1.0 + stop_grace_period: 15s + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + interval: 5s + timeout: 4s + retries: 5 + start_period: 15s + + api2: + build: . + image: mtusi-messenger-api:4.0.0 + environment: *api-env + depends_on: + migrate: {condition: service_completed_successfully} + s3-bootstrap: {condition: service_completed_successfully} + db: {condition: service_started} + redis: {condition: service_started} + mem_limit: 512m + cpus: 1.0 + stop_grace_period: 15s + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + interval: 5s + timeout: 4s + retries: 5 + start_period: 15s + + migrate: + build: . + image: mtusi-messenger-api:4.0.0 + command: ["alembic", "upgrade", "head"] + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?Create a local .env first}@db:5432/${POSTGRES_DB:-messenger} + depends_on: + db: {condition: service_started} + restart: "no" + + lb: + image: nginx:1.27.5-alpine + ports: + - "${APP_PORT:-8080}:8080" + volumes: + - ./infra/nginx.conf:/etc/nginx/conf.d/default.conf:ro + depends_on: [api1, api2] + mem_limit: 128m + cpus: 0.5 + restart: unless-stopped + + db: + image: postgres:16.4-alpine + environment: + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Create a local .env first} + POSTGRES_DB: ${POSTGRES_DB:-messenger} + volumes: ["postgres_data:/var/lib/postgresql/data"] + mem_limit: 1g + cpus: 1.0 + restart: unless-stopped + + redis: + image: redis:7.4.2-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"] + volumes: ["redis_data:/data"] + mem_limit: 256m + cpus: 0.5 + restart: unless-stopped + + minio: + image: dappros/minio:RELEASE.2025-09-07T16-13-09Z + command: ["server", "/data", "--address", ":9000", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Create local MinIO credentials} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Create local MinIO credentials} + volumes: ["minio_data:/data"] + expose: ["9000"] + mem_limit: 768m + cpus: 1.0 + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:9000/minio/health/ready"] + interval: 5s + timeout: 3s + retries: 10 + + s3-bootstrap: + image: ghcr.io/golithus/mc:RELEASE.2025-08-13T08-35-41Z@sha256:dde55ccc0bc9e65dc582c8e652d652eea1438c6e60d774524f4da15d7b26a900 + entrypoint: ["/bin/sh", "/bootstrap.sh"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Create local MinIO credentials} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Create local MinIO credentials} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Create local S3 credentials} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Create local S3 credentials} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + volumes: ["./infra/minio-bootstrap.sh:/bootstrap.sh:ro"] + depends_on: + minio: {condition: service_healthy} + restart: "no" + +volumes: + postgres_data: + redis_data: + minio_data: diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 00000000..f8e3a2c2 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu +python - <<'PY' +import os, time +from sqlalchemy import create_engine, text +url = os.environ["DATABASE_URL"] +engine = create_engine(url, connect_args={"connect_timeout": 3}) +for attempt in range(60): + try: + with engine.connect() as connection: + connection.execute(text("SELECT 1")) + break + except Exception: + if attempt == 59: + raise + time.sleep(1) +PY +exec "$@" diff --git a/evidence/failover-actions.txt b/evidence/failover-actions.txt new file mode 100644 index 00000000..60df538b --- /dev/null +++ b/evidence/failover-actions.txt @@ -0,0 +1,7 @@ +2026-10-30 18:09:26 +03:00 target=96571e19-0cb0-44e2-afac-ecbb764c0231 service=api1 container=/mtusi2026lab4-fork-api1-1 +2026-10-30 18:10:45 +03:00 target=311e027f-530a-46ed-819e-c517a4dd920c service=api1 container=/mtusi2026lab4-fork-api1-1 +2026-10-30 18:11:21 +03:00 target=4e8e825c-f169-4cac-968f-e55bda6f9d40 service=api2 container=/mtusi2026lab4-fork-api2-1 +2026-10-30 18:12:31 +03:00 target=7147cc32-6b4b-4d95-9b19-3c5cbc34261c service=api2 container=/mtusi2026lab4-fork-api2-1 +2026-10-30 18:13:19 +03:00 target=19a0efe6-875e-4400-84fe-e6c4a00f81db service=api2 container=/mtusi2026lab4-fork-api2-1 +2026-10-30 18:21:48 +03:00 target=953498b4-1d5a-4ad1-9e5e-6b249db33c27 service=api2 container=/mtusi2026lab4-fork-api2-1 +2026-10-30 19:44:36 +03:00 target=e5a8a25b-7661-4278-8b65-c82c107edd63 service=api2 container=/mtusi2026lab4-fork-api2-1 diff --git a/evidence/persistence-actions.txt b/evidence/persistence-actions.txt new file mode 100644 index 00000000..ea60b697 --- /dev/null +++ b/evidence/persistence-actions.txt @@ -0,0 +1,39 @@ +[2026-10-30 18:09:16 +03:00] +before service=db container=/mtusi2026lab4-fork-db-1 started_at=2026-10-30T15:05:20.153773062Z +before service=redis container=/mtusi2026lab4-fork-redis-1 started_at=2026-10-30T15:05:20.005401544Z +before service=minio container=/mtusi2026lab4-fork-minio-1 started_at=2026-10-30T15:05:20.209818194Z +before service=api1 container=/mtusi2026lab4-fork-api1-1 started_at=2026-10-30T15:06:01.900526354Z +before service=api2 container=/mtusi2026lab4-fork-api2-1 started_at=2026-10-30T15:06:02.05116246Z +before service=lb container=/mtusi2026lab4-fork-lb-1 started_at=2026-10-30T15:06:02.686187219Z +after service=db container=/mtusi2026lab4-fork-db-1 started_at=2026-10-30T15:09:10.254043125Z +after service=redis container=/mtusi2026lab4-fork-redis-1 started_at=2026-10-30T15:09:09.117211066Z +after service=minio container=/mtusi2026lab4-fork-minio-1 started_at=2026-10-30T15:09:09.890230629Z +after service=api1 container=/mtusi2026lab4-fork-api1-1 started_at=2026-10-30T15:09:10.711312133Z +after service=api2 container=/mtusi2026lab4-fork-api2-1 started_at=2026-10-30T15:09:10.850005929Z +after service=lb container=/mtusi2026lab4-fork-lb-1 started_at=2026-10-30T15:09:10.824404805Z +[2026-10-30 18:21:45 +03:00] +before service=db container=/mtusi2026lab4-fork-db-1 started_at=2026-10-30T15:09:10.254043125Z +before service=redis container=/mtusi2026lab4-fork-redis-1 started_at=2026-10-30T15:09:09.117211066Z +before service=minio container=/mtusi2026lab4-fork-minio-1 started_at=2026-10-30T15:20:27.258000464Z +before service=api1 container=/mtusi2026lab4-fork-api1-1 started_at=2026-10-30T15:11:12.135947013Z +before service=api2 container=/mtusi2026lab4-fork-api2-1 started_at=2026-10-30T15:13:58.095302016Z +before service=lb container=/mtusi2026lab4-fork-lb-1 started_at=2026-10-30T15:19:44.692767393Z +after service=db container=/mtusi2026lab4-fork-db-1 started_at=2026-10-30T15:21:39.637912454Z +after service=redis container=/mtusi2026lab4-fork-redis-1 started_at=2026-10-30T15:21:39.288384494Z +after service=minio container=/mtusi2026lab4-fork-minio-1 started_at=2026-10-30T15:21:38.899261397Z +after service=api1 container=/mtusi2026lab4-fork-api1-1 started_at=2026-10-30T15:21:39.91288599Z +after service=api2 container=/mtusi2026lab4-fork-api2-1 started_at=2026-10-30T15:21:39.838501089Z +after service=lb container=/mtusi2026lab4-fork-lb-1 started_at=2026-10-30T15:21:39.478746319Z +[2026-10-30 19:44:33 +03:00] +before service=db container=/mtusi2026lab4-fork-db-1 started_at=2026-10-30T15:21:39.637912454Z +before service=redis container=/mtusi2026lab4-fork-redis-1 started_at=2026-10-30T15:21:39.288384494Z +before service=minio container=/mtusi2026lab4-fork-minio-1 started_at=2026-10-30T16:44:09.459051232Z +before service=api1 container=/mtusi2026lab4-fork-api1-1 started_at=2026-10-30T15:21:39.91288599Z +before service=api2 container=/mtusi2026lab4-fork-api2-1 started_at=2026-10-30T15:22:04.566204453Z +before service=lb container=/mtusi2026lab4-fork-lb-1 started_at=2026-10-30T15:21:39.478746319Z +after service=db container=/mtusi2026lab4-fork-db-1 started_at=2026-10-30T16:44:28.098665492Z +after service=redis container=/mtusi2026lab4-fork-redis-1 started_at=2026-10-30T16:44:26.989436959Z +after service=minio container=/mtusi2026lab4-fork-minio-1 started_at=2026-10-30T16:44:27.865345659Z +after service=api1 container=/mtusi2026lab4-fork-api1-1 started_at=2026-10-30T16:44:28.223433032Z +after service=api2 container=/mtusi2026lab4-fork-api2-1 started_at=2026-10-30T16:44:28.295821672Z +after service=lb container=/mtusi2026lab4-fork-lb-1 started_at=2026-10-30T16:44:28.09544913Z diff --git a/infra/minio-bootstrap.sh b/infra/minio-bootstrap.sh new file mode 100644 index 00000000..5410b491 --- /dev/null +++ b/infra/minio-bootstrap.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu +until mc alias set local http://minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do sleep 2; done +mc mb --ignore-existing "local/$S3_BUCKET" +mc anonymous set none "local/$S3_BUCKET" +if ! mc admin user info local "$S3_ACCESS_KEY" >/dev/null 2>&1; then + mc admin user add local "$S3_ACCESS_KEY" "$S3_SECRET_KEY" +fi +cat >/tmp/attachment-policy.json </dev/null 2>&1; then + mc admin policy create local attachments-api /tmp/attachment-policy.json +fi +mc admin policy attach local attachments-api --user "$S3_ACCESS_KEY" diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 00000000..361e4197 --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,42 @@ +upstream api_backend { + zone api_backend 64k; + resolver 127.0.0.11 valid=2s ipv6=off; + server api1:8080 resolve max_fails=1 fail_timeout=2s; + server api2:8080 resolve max_fails=1 fail_timeout=2s; +} + +server { + listen 8080; + # Leave a small margin so the API returns its documented JSON 413 at 10 MiB. + client_max_body_size 11m; + client_body_timeout 20s; + proxy_connect_timeout 1s; + proxy_send_timeout 20s; + proxy_read_timeout 20s; + error_page 502 504 = @upstream_unavailable; + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Request-Id $request_id; + proxy_intercept_errors on; + # Retry GET/HEAD on a failed backend. Nginx never replays a POST after + # its request has been sent unless `non_idempotent` is enabled; uploads + # therefore remain single-shot, while message retries stay client-led. + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 5s; + } + + # Keep the public API error contract when all upstreams are unavailable. + location @upstream_unavailable { + default_type application/json; + add_header X-Request-Id $request_id always; + add_header X-Instance-Id "lb" always; + return 503 '{"error":{"code":"unavailable","message":"API upstream unavailable","request_id":"$request_id"}}'; + } +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..5cc9dfd5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.109.0 +uvicorn[standard]==0.27.0 +pydantic==2.5.3 +sqlalchemy==2.0.25 +alembic==1.13.1 +psycopg2-binary==2.9.9 +redis==5.0.1 +boto3==1.35.99 +Pillow==11.0.0 diff --git a/scripts/restart.bat b/scripts/restart.bat new file mode 100644 index 00000000..21fe545b --- /dev/null +++ b/scripts/restart.bat @@ -0,0 +1,4 @@ +@echo off +setlocal +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0restart.ps1" +exit /b %errorlevel% diff --git a/scripts/restart.ps1 b/scripts/restart.ps1 new file mode 100644 index 00000000..349ceeb7 --- /dev/null +++ b/scripts/restart.ps1 @@ -0,0 +1,21 @@ +$ErrorActionPreference = 'Stop' +Set-Location (Join-Path $PSScriptRoot '..') +function Get-ContainerState($phase) { + foreach ($service in @('db', 'redis', 'minio', 'api1', 'api2', 'lb')) { + $containers = docker compose --env-file .env ps -q $service + if ($LASTEXITCODE -ne 0) { throw "Cannot list $service containers" } + foreach ($container in $containers) { + $name = docker inspect --format '{{.Name}}' $container + $started = docker inspect --format '{{.State.StartedAt}}' $container + "$phase service=$service container=$name started_at=$started" + } + } +} +$before = @(Get-ContainerState 'before') +docker compose --env-file .env restart db redis minio api1 api2 lb | Out-Null +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +$after = @(Get-ContainerState 'after') +$stamp = Get-Date -Format 'yyyy-MM-dd HH:mm:ss K' +Add-Content -Path (Join-Path $PSScriptRoot '..\evidence\persistence-actions.txt') -Value "[$stamp]" +Add-Content -Path (Join-Path $PSScriptRoot '..\evidence\persistence-actions.txt') -Value $before +Add-Content -Path (Join-Path $PSScriptRoot '..\evidence\persistence-actions.txt') -Value $after diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..7422f465 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env restart db redis minio api1 api2 lb diff --git a/scripts/stop-one.bat b/scripts/stop-one.bat new file mode 100644 index 00000000..e2c0c17e --- /dev/null +++ b/scripts/stop-one.bat @@ -0,0 +1,4 @@ +@echo off +setlocal +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0stop-one.ps1" +exit /b %errorlevel% diff --git a/scripts/stop-one.ps1 b/scripts/stop-one.ps1 new file mode 100644 index 00000000..f96c9ecd --- /dev/null +++ b/scripts/stop-one.ps1 @@ -0,0 +1,22 @@ +$ErrorActionPreference = 'Stop' +Set-Location (Join-Path $PSScriptRoot '..') +$target = $env:TARGET_INSTANCE_ID +if ([string]::IsNullOrWhiteSpace($target)) { exit 2 } +foreach ($service in @('api1', 'api2')) { + $containers = docker compose --env-file .env ps -q $service + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + foreach ($container in $containers) { + try { + $instance = docker exec $container python -c "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).headers['X-Instance-Id'])" 2>$null + if ($LASTEXITCODE -eq 0 -and $instance.Trim() -eq $target) { + $name = docker inspect --format '{{.Name}}' $container + $stamp = Get-Date -Format 'yyyy-MM-dd HH:mm:ss K' + Add-Content -Path (Join-Path $PSScriptRoot '..\evidence\failover-actions.txt') -Value "$stamp target=$target service=$service container=$name" + docker stop --time 5 $container | Out-Null + exit $LASTEXITCODE + } + } catch { } + } +} +Write-Error 'No running API container matched TARGET_INSTANCE_ID' +exit 1 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..87d71e3e --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu +target=${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required} +cd "$(dirname "$0")/.." +for service in api1 api2; do + for container in $(docker compose --env-file .env ps -q "$service"); do + instance=$(docker exec "$container" python -c 'import urllib.request; r=urllib.request.urlopen("http://127.0.0.1:8080/health/live",timeout=2); print(r.headers["X-Instance-Id"])' 2>/dev/null || true) + if [ "$instance" = "$target" ]; then + docker stop --time 5 "$container" >/dev/null + exit 0 + fi + done +done +echo "No running API container matched TARGET_INSTANCE_ID" >&2 +exit 1 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..fec0e09e --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""MTUSI Messenger service package.""" diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 00000000..02a49ee1 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,31 @@ +"""scrypt password KDF and opaque, cryptographically random bearer tokens.""" +import base64 +import hashlib +import hmac +import os +import secrets + +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) + + +def hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + digest = hashlib.scrypt(password.encode("utf-8"), salt=salt, n=2**14, r=8, p=1, dklen=32) + return "scrypt$16384$8$1$" + base64.urlsafe_b64encode(salt + digest).decode("ascii") + + +def verify_password(password: str, encoded: str) -> bool: + try: + scheme, n, r, p, raw = encoded.split("$", 4) + if scheme != "scrypt": + return False + data = base64.urlsafe_b64decode(raw.encode("ascii")) + salt, expected = data[:16], data[16:] + actual = hashlib.scrypt(password.encode("utf-8"), salt=salt, n=int(n), r=int(r), p=int(p), dklen=len(expected)) + return hmac.compare_digest(actual, expected) + except (ValueError, TypeError): + return False + + +def new_token() -> str: + return secrets.token_urlsafe(32) diff --git a/src/database.py b/src/database.py new file mode 100644 index 00000000..17bb2ea5 --- /dev/null +++ b/src/database.py @@ -0,0 +1,97 @@ +"""PostgreSQL schema and bounded connection pool.""" +import os +import uuid +from datetime import datetime, timezone +from sqlalchemy import (Column, DateTime, ForeignKey, Index, String, Text, Integer, JSON, + UniqueConstraint, create_engine) +from sqlalchemy.orm import declarative_base, relationship, sessionmaker + +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@db:5432/messenger") +engine = create_engine(DATABASE_URL, pool_size=5, max_overflow=5, pool_timeout=5, + pool_pre_ping=True, connect_args={"connect_timeout": 5, + "options": "-c statement_timeout=5000"}) +SessionLocal = sessionmaker(bind=engine, expire_on_commit=False) +Base = declarative_base() +now = lambda: datetime.now(timezone.utc) + + +class User(Base): + __tablename__ = "users" + id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + username = Column(String(32), nullable=False) + display_name = Column(String(100), nullable=False) + password_hash = Column(String(255), nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False, default=now) + __table_args__ = (Index("ix_users_username", "username", unique=True),) + + +class Session(Base): + __tablename__ = "sessions" + id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + token = Column(String(255), nullable=False) + user_id = Column(String, ForeignKey("users.id"), nullable=False) + expires_at = Column(DateTime(timezone=True), nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False, default=now) + user = relationship("User") + __table_args__ = (Index("ix_sessions_token", "token", unique=True),) + + +class Chat(Base): + __tablename__ = "chats" + id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + title = Column(String(100), nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False, default=now) + members = relationship("ChatMember", back_populates="chat", cascade="all, delete-orphan") + + +class ChatMember(Base): + __tablename__ = "chat_members" + id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + chat_id = Column(String, ForeignKey("chats.id"), nullable=False) + user_id = Column(String, ForeignKey("users.id"), nullable=False) + chat = relationship("Chat", back_populates="members") + user = relationship("User") + __table_args__ = (UniqueConstraint("chat_id", "user_id", name="uq_chat_member"), + Index("ix_chat_members_user_chat", "user_id", "chat_id")) + + +class Message(Base): + __tablename__ = "messages" + id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + chat_id = Column(String, ForeignKey("chats.id"), nullable=False) + sender_id = Column(String, ForeignKey("users.id"), nullable=False) + text = Column(Text, nullable=False) + attachment_ids = Column(JSON, nullable=False, default=list) + created_at = Column(DateTime(timezone=True), nullable=False, default=now) + chat = relationship("Chat") + sender = relationship("User") + __table_args__ = (Index("ix_messages_chat_created_id", "chat_id", "created_at", "id"),) + + +class Attachment(Base): + __tablename__ = "attachments" + id = Column(String, primary_key=True) + chat_id = Column(String, ForeignKey("chats.id"), nullable=False, index=True) + owner_id = Column(String, ForeignKey("users.id"), nullable=False) + filename = Column(String(255), nullable=False) + content_type = Column(String(32), nullable=False) + size_bytes = Column(Integer, nullable=False) + sha256 = Column(String(64), nullable=False) + object_key = Column(String(128), nullable=False, unique=True) + status = Column(String(16), nullable=False, default="ready") + created_at = Column(DateTime(timezone=True), nullable=False, default=now) + updated_at = Column(DateTime(timezone=True), nullable=False, default=now) + + +class MessageIdempotency(Base): + __tablename__ = "message_idempotency" + id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + user_id = Column(String, ForeignKey("users.id"), nullable=False) + method = Column(String(8), nullable=False) + chat_path = Column(String(80), nullable=False) + key = Column(String(128), nullable=False) + request_hash = Column(String(64), nullable=False) + message_id = Column(String, ForeignKey("messages.id"), nullable=True) + created_at = Column(DateTime(timezone=True), nullable=False, default=now) + __table_args__ = (UniqueConstraint("user_id", "method", "chat_path", "key", + name="uq_idempotency_scope"),) diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..910b56b5 --- /dev/null +++ b/src/main.py @@ -0,0 +1,361 @@ +"""Lab 4 REST API: shared messenger state and private S3 attachments.""" +import base64 +import binascii +import hashlib +import json +import re +import uuid +import hashlib +import tempfile +import re as regex +from datetime import timezone +from fastapi import FastAPI, Request, Query +from fastapi.responses import JSONResponse, StreamingResponse +from starlette.concurrency import run_in_threadpool +from PIL import Image, UnidentifiedImageError +from botocore.exceptions import BotoCoreError, ClientError +from fastapi.exceptions import RequestValidationError +from sqlalchemy import text +from sqlalchemy.exc import SQLAlchemyError +from .database import SessionLocal +from .models import CreateUser, CreateChat, CreateMessage +from .store import store, cursor_for_chat, cursor_for_message +from . import session_cache +from . import object_store + + +class AppError(Exception): + def __init__(self, status, code, message, headers=None): + self.status, self.code, self.message, self.headers = status, code, message, headers or {} + + +app = FastAPI(title="MTUSI Messenger", version="4.0.0") +INSTANCE_ID = str(uuid.uuid4()) + + +@app.middleware("http") +async def request_headers(request: Request, call_next): + request.state.request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4()) + response = await call_next(request) + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(AppError) +async def app_error_handler(request, exc): + headers = {"X-Request-Id": request.state.request_id, **exc.headers} + headers["X-Instance-Id"] = INSTANCE_ID + return JSONResponse(status_code=exc.status, content={"error":{"code":exc.code,"message":exc.message,"request_id":request.state.request_id}}, headers=headers) + + +@app.exception_handler(RequestValidationError) +async def validation_handler(request, exc): + return JSONResponse(status_code=400, content={"error":{"code":"invalid_request","message":"Invalid request","request_id":request.state.request_id}}, headers={"X-Request-Id":request.state.request_id,"X-Instance-Id":INSTANCE_ID}) + + +@app.exception_handler(SQLAlchemyError) +async def database_error_handler(request, exc): + return JSONResponse(status_code=503, content={"error":{"code":"unavailable","message":"Database unavailable","request_id":request.state.request_id}}, headers={"X-Request-Id":request.state.request_id,"X-Instance-Id":INSTANCE_ID}) + + +@app.exception_handler(Exception) +async def internal_handler(request, exc): + return JSONResponse(status_code=500, content={"error":{"code":"internal_error","message":"Internal server error","request_id":request.state.request_id}}, headers={"X-Request-Id":request.state.request_id,"X-Instance-Id":INSTANCE_ID}) + + +def stamp(value): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def user_json(user): + return {"id":user.id,"username":user.username,"display_name":user.display_name,"created_at":stamp(user.created_at)} + + +def auth_user(request): + header = request.headers.get("Authorization", "") + if not header.startswith("Bearer ") or not header[7:]: + raise AppError(401,"unauthorized","Authentication required",{"WWW-Authenticate":'Bearer realm="messenger"'}) + try: + found = store.get_session_by_token(header[7:]) + except session_cache.CacheUnavailable: + raise AppError(503,"unavailable","Session cache unavailable") + except SQLAlchemyError: + raise AppError(503,"unavailable","Authentication store unavailable") + if not found: + raise AppError(401,"unauthorized","Invalid or expired session",{"WWW-Authenticate":'Bearer realm="messenger"'}) + return found[0] + + +def chat_json(chat): + return {"id":chat.id,"title":chat.title,"member_ids":[m.user_id for m in chat.members],"created_at":stamp(chat.created_at)} + + +def validate_uuid(value): + try: + uuid.UUID(value) + except (ValueError, AttributeError): + raise AppError(400,"invalid_request","Invalid UUID") + + +@app.get("/health/live") +async def live(): + return {"status":"ok"} + + +@app.get("/health/ready") +async def ready(): + try: + with SessionLocal() as db: + db.execute(text("SELECT 1")) + session_cache.ping() + object_store.s3.list_objects_v2(Bucket=object_store.S3_BUCKET, Prefix="original/", MaxKeys=1) + return {"status":"ok"} + except session_cache.CacheUnavailable: + raise AppError(503,"unavailable","Session cache unavailable") + except SQLAlchemyError: + raise AppError(503,"unavailable","Database unavailable") + except (BotoCoreError, ClientError): + raise AppError(503,"unavailable","Object storage unavailable") + + +@app.post("/api/v1/users", status_code=201) +async def register(data: CreateUser): + if not data.display_name.strip() or not 12 <= len(data.password) <= 128: + raise AppError(400,"invalid_request","Invalid user fields") + try: + return user_json(store.add_user(data.username,data.display_name,data.password)) + except ValueError as exc: + if str(exc) == "USERNAME_EXISTS": + raise AppError(409,"conflict","Username already exists") + raise AppError(400,"invalid_request","Invalid user fields") + except SQLAlchemyError: + raise AppError(503,"unavailable","Database unavailable") + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def login(request: Request): + header = request.headers.get("Authorization", "") + challenge = {'WWW-Authenticate':'Basic realm="messenger", charset="UTF-8"'} + if not header.startswith("Basic "): + raise AppError(401,"unauthorized","Invalid credentials",challenge) + try: + decoded = base64.b64decode(header[6:], validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (binascii.Error, UnicodeError, ValueError): + raise AppError(401,"unauthorized","Invalid credentials",challenge) + try: + user = store.verify_user_password(username,password) + if not user: + raise AppError(401,"unauthorized","Invalid credentials",challenge) + session = store.create_session(user.id) + return {"token":session.token,"token_type":"Bearer","expires_at":stamp(session.expires_at),"user":user_json(user)} + except session_cache.CacheUnavailable: + raise AppError(503,"unavailable","Session cache unavailable") + except SQLAlchemyError: + raise AppError(503,"unavailable","Database unavailable") + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def logout(request: Request): + auth_user(request) + try: + store.delete_session(request.headers["Authorization"][7:]) + except session_cache.CacheUnavailable: + raise AppError(503,"unavailable","Session cache unavailable") + except SQLAlchemyError: + raise AppError(503,"unavailable","Authentication store unavailable") + return None + + +@app.get("/api/v1/users/me") +async def me(request: Request): + user_id = auth_user(request) + user = store.get_user_by_id(user_id) + if not user: + raise AppError(401,"unauthorized","Invalid session",{'WWW-Authenticate':'Bearer realm="messenger"'}) + return user_json(user) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(data: CreateChat, request: Request): + user_id = auth_user(request) + if not data.title.strip(): + raise AppError(400,"invalid_request","Invalid title") + try: + return chat_json(store.create_chat(data.title,data.member_ids,user_id)) + except ValueError as exc: + if str(exc)=="USER_NOT_FOUND": + raise AppError(400,"invalid_request","Unknown member") + raise AppError(400,"invalid_request","Invalid members") + except SQLAlchemyError: + raise AppError(503,"unavailable","Database unavailable") + + +@app.get("/api/v1/chats") +async def list_chats(request: Request, limit: int = Query(50,ge=1,le=100), cursor: str|None=None): + user_id=auth_user(request) + try: + rows=store.get_user_chats(user_id,cursor,limit+1) + except ValueError: + raise AppError(400,"invalid_request","Invalid cursor") + more=len(rows)>limit + rows=rows[:limit] + return {"items":[chat_json(c) for c in rows],"next_cursor":cursor_for_chat(rows[-1]) if more else None} + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, request: Request): + user_id=auth_user(request) + validate_uuid(chat_id) + if not store.is_chat_member(chat_id,user_id): + raise AppError(404,"not_found","Chat not found") + chat=store.get_chat(chat_id) + if not chat: + raise AppError(404,"not_found","Chat not found") + return chat_json(chat) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(chat_id: str, data: CreateMessage, request: Request): + user_id=auth_user(request) + validate_uuid(chat_id) + if data.text is None or len(data.text)>4000 or len(data.attachment_ids) != len(set(data.attachment_ids)): + raise AppError(400,"invalid_request","Invalid message text") + for attachment_id in data.attachment_ids: + validate_uuid(attachment_id) + if not data.text.strip() and not data.attachment_ids: + raise AppError(400,"invalid_request","Message must contain text or attachments") + key = request.headers.get("Idempotency-Key", "") + if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key, flags=re.ASCII): + raise AppError(400,"invalid_request","A valid Idempotency-Key is required") + canonical = json.dumps({"text":data.text,"attachment_ids":data.attachment_ids}, ensure_ascii=False, + sort_keys=True, separators=(",", ":")) + request_hash = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + try: + msg=store.add_message_idempotent(chat_id,user_id,data.text,data.attachment_ids,key,request_hash) + return {"id":msg.id,"chat_id":msg.chat_id,"sender_id":msg.sender_id,"text":msg.text,"created_at":stamp(msg.created_at),"attachment_ids":msg.attachment_ids} + except ValueError as exc: + if str(exc) == "IDEMPOTENCY_CONFLICT": + raise AppError(409,"conflict","Idempotency key was used with a different request") + if str(exc) == "EMPTY_MESSAGE": + raise AppError(400,"invalid_request","Message must contain text or attachments") + raise AppError(404,"not_found","Chat not found") + except SQLAlchemyError: + raise AppError(503,"unavailable","Database unavailable") + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(chat_id: str, request: Request, limit: int=Query(50,ge=1,le=100), cursor: str|None=None): + user_id=auth_user(request) + validate_uuid(chat_id) + if not store.is_chat_member(chat_id,user_id): + raise AppError(404,"not_found","Chat not found") + try: + rows=store.get_messages(chat_id,cursor,limit+1) + except ValueError: + raise AppError(400,"invalid_request","Invalid cursor") + more=len(rows)>limit + rows=rows[:limit] + items=[{"id":m.id,"chat_id":m.chat_id,"sender_id":m.sender_id,"text":m.text,"created_at":stamp(m.created_at),"attachment_ids":m.attachment_ids or []} for m in rows] + return {"items":items,"next_cursor":cursor_for_message(rows[-1]) if more else None} + + +def attachment_json(row): + return {"id":row.id,"chat_id":row.chat_id,"owner_id":row.owner_id,"filename":row.filename, + "content_type":row.content_type,"size_bytes":row.size_bytes,"sha256":row.sha256, + "status":"ready","variants":[],"error_code":None, + "created_at":stamp(row.created_at),"updated_at":stamp(row.updated_at)} + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment(chat_id: str, request: Request, filename: str = Query(...)): + user_id = auth_user(request) + validate_uuid(chat_id) + if not store.is_chat_member(chat_id, user_id): + raise AppError(404, "not_found", "Chat not found") + if not 1 <= len(filename) <= 255 or any(ch in filename for ch in "/\\\x00") or any(ord(ch) < 32 or 127 <= ord(ch) < 160 for ch in filename): + raise AppError(400, "invalid_request", "Invalid filename") + declared = request.headers.get("content-type", "").split(";", 1)[0].lower() + if declared not in ("image/png", "image/jpeg"): + raise AppError(415, "unsupported_media_type", "Only PNG and JPEG are accepted") + max_bytes = 10 * 1024 * 1024 + spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b") + digest = hashlib.sha256() + size = 0 + try: + async for chunk in request.stream(): + size += len(chunk) + if size > max_bytes: + raise AppError(413, "payload_too_large", "Image exceeds 10 MiB") + digest.update(chunk) + spool.write(chunk) + if size == 0: + raise AppError(415, "unsupported_media_type", "Empty image") + spool.seek(0) + try: + image = Image.open(spool) + fmt = image.format + width, height = image.size + if width * height > 40_000_000: + raise AppError(413, "payload_too_large", "Image exceeds 40 million pixels") + if (fmt, declared) not in (("PNG", "image/png"), ("JPEG", "image/jpeg")): + raise AppError(415, "unsupported_media_type", "Image content does not match Content-Type") + image.verify() + except AppError: + raise + except (UnidentifiedImageError, OSError, Image.DecompressionBombError, Image.DecompressionBombWarning): + raise AppError(415, "unsupported_media_type", "Invalid or unsupported image") + object_id = str(uuid.uuid4()) + object_key = f"original/{object_id}" + spool.seek(0) + await run_in_threadpool(object_store.s3.put_object, Bucket=object_store.S3_BUCKET, + Key=object_key, Body=spool, ContentLength=size, ContentType=declared) + row = store.save_attachment(dict(id=object_id, chat_id=chat_id, owner_id=user_id, + filename=filename, content_type=declared, size_bytes=size, sha256=digest.hexdigest(), + object_key=object_key, status="ready")) + return attachment_json(row) + except AppError: + raise + except (BotoCoreError, ClientError): + raise AppError(503, "unavailable", "Object storage unavailable") + finally: + spool.close() + + +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment(attachment_id: str, request: Request): + user_id = auth_user(request) + validate_uuid(attachment_id) + row = store.get_attachment(attachment_id, user_id) + if not row: + raise AppError(404, "not_found", "Attachment not found") + return attachment_json(row) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(attachment_id: str, request: Request, variant: str = "original"): + user_id = auth_user(request) + validate_uuid(attachment_id) + row = store.get_attachment(attachment_id, user_id) + if not row: + raise AppError(404, "not_found", "Attachment not found") + if variant != "original": + raise AppError(404, "not_found", "Attachment variant not found") + try: + obj = await run_in_threadpool(object_store.s3.get_object, Bucket=object_store.S3_BUCKET, Key=row.object_key) + except (BotoCoreError, ClientError): + raise AppError(503, "unavailable", "Object storage unavailable") + body = obj["Body"] + def chunks(): + try: + while True: + chunk = body.read(64 * 1024) + if not chunk: + break + yield chunk + finally: + body.close() + return StreamingResponse(chunks(), media_type=row.content_type, headers={"Content-Length": str(row.size_bytes)}) diff --git a/src/models.py b/src/models.py new file mode 100644 index 00000000..beae6a43 --- /dev/null +++ b/src/models.py @@ -0,0 +1,28 @@ +from pydantic import BaseModel, Field, ConfigDict +from typing import List + + +# ==================== REQUEST MODELS ==================== + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid", str_strip_whitespace=False) + username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(..., min_length=1, max_length=100) + password: str = Field(..., min_length=12, max_length=128) + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str = Field(..., min_length=1, max_length=100) + member_ids: List[str] = Field(..., min_length=1, max_length=100) + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + text: str = Field(..., max_length=4000) + attachment_ids: List[str] = Field(default_factory=list, max_length=10) + + +class LoginRequest(BaseModel): + username: str + password: str diff --git a/src/object_store.py b/src/object_store.py new file mode 100644 index 00000000..8b9a74ba --- /dev/null +++ b/src/object_store.py @@ -0,0 +1,14 @@ +"""Private S3-compatible object storage client.""" +import os +import boto3 +from botocore.config import Config + +S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") +s3 = boto3.client( + "s3", + endpoint_url=os.environ.get("S3_ENDPOINT", "http://minio:9000"), + aws_access_key_id=os.environ["S3_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_SECRET_KEY"], + region_name=os.environ.get("S3_REGION", "us-east-1"), + config=Config(connect_timeout=3, read_timeout=10, retries={"max_attempts": 2}), +) diff --git a/src/session_cache.py b/src/session_cache.py new file mode 100644 index 00000000..b17a9720 --- /dev/null +++ b/src/session_cache.py @@ -0,0 +1,72 @@ +"""Redis TTL cache for active sessions; PostgreSQL remains authoritative.""" +import os +import socket +from urllib.parse import urlparse +from redis import Redis +from redis.backoff import NoBackoff +from redis.exceptions import RedisError +from redis.retry import Retry + +CACHE_URL = os.getenv("CACHE_URL", "redis://redis:6379/0") +_cache_address = urlparse(CACHE_URL) +_cache_host = _cache_address.hostname or "redis" +_cache_port = _cache_address.port or (6380 if _cache_address.scheme == "rediss" else 6379) +try: + _cache_ips = tuple(dict.fromkeys(item[4][0] for item in socket.getaddrinfo( + _cache_host, _cache_port, type=socket.SOCK_STREAM))) +except OSError: + _cache_ips = (_cache_host,) +client = Redis.from_url(CACHE_URL, decode_responses=True, + socket_connect_timeout=0.2, socket_timeout=0.2, + health_check_interval=15, retry=Retry(NoBackoff(), 0)) + + +class CacheUnavailable(Exception): + """The required session cache cannot be reached within its short timeout.""" + + +def _check_port(): + # Fail fast when the Redis container is stopped. redis-py may otherwise + # spend several seconds cycling through connection retries on Docker DNS. + try: + for address in _cache_ips: + try: + with socket.create_connection((address, _cache_port), timeout=0.2): + return + except OSError: + continue + raise OSError("Redis is not accepting connections") + except OSError as exc: + raise CacheUnavailable from exc + + +def get(token: str): + try: + _check_port() + return client.get("session:" + token) + except RedisError as exc: + raise CacheUnavailable from exc + + +def put(token: str, user_id: str, ttl_seconds: int): + try: + _check_port() + client.set("session:" + token, user_id, ex=max(1, ttl_seconds)) + except RedisError as exc: + raise CacheUnavailable from exc + + +def delete(token: str): + try: + _check_port() + client.delete("session:" + token) + except RedisError as exc: + raise CacheUnavailable from exc + + +def ping(): + try: + _check_port() + return bool(client.ping()) + except RedisError as exc: + raise CacheUnavailable from exc diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..3ad1c097 --- /dev/null +++ b/src/store.py @@ -0,0 +1,209 @@ +"""Small repository layer; each operation owns and closes its DB transaction.""" +import base64 +import json +import uuid +from datetime import datetime, timezone, timedelta +import math +from sqlalchemy import and_, or_, select, update +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.orm import selectinload +from sqlalchemy.exc import IntegrityError +from .database import User, Session as SessionRow, Chat, ChatMember, Message, MessageIdempotency, Attachment, SessionLocal +from .auth import hash_password, verify_password, new_token, SESSION_TTL_SECONDS +from . import session_cache + + +class Store: + def add_user(self, username, display_name, password): + with SessionLocal() as db: + user = User(username=username, display_name=display_name, password_hash=hash_password(password)) + db.add(user) + try: + db.commit() + except IntegrityError: + db.rollback() + raise ValueError("USERNAME_EXISTS") + db.refresh(user) + return user + + def get_user_by_id(self, user_id): + with SessionLocal() as db: + return db.query(User).filter(User.id == user_id).first() + + def verify_user_password(self, username, password): + with SessionLocal() as db: + user = db.query(User).filter(User.username == username).first() + # A fixed dummy hash is avoided here; the public contract only requires + # identical response status and challenge for unknown users/bad passwords. + if not user or not verify_password(password, user.password_hash): + return None + return user + + def create_session(self, user_id): + with SessionLocal() as db: + row = SessionRow(token=new_token(), user_id=user_id, + expires_at=datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS)) + db.add(row) + db.commit() + db.refresh(row) + try: + session_cache.put(row.token, row.user_id, SESSION_TTL_SECONDS) + except session_cache.CacheUnavailable: + db.delete(row) + db.commit() + raise + return row + + def get_session_by_token(self, token): + cached_user_id = session_cache.get(token) + if cached_user_id: + return (cached_user_id, None) + with SessionLocal() as db: + row = db.query(SessionRow).filter(SessionRow.token == token, + SessionRow.expires_at > datetime.now(timezone.utc)).first() + if not row: + return None + remaining = math.ceil((row.expires_at - datetime.now(timezone.utc)).total_seconds()) + session_cache.put(row.token, row.user_id, remaining) + return (row.user_id, row.expires_at) + + def delete_session(self, token): + # Require successful cache invalidation before reporting a completed logout. + session_cache.delete(token) + with SessionLocal() as db: + row = db.query(SessionRow).filter(SessionRow.token == token).first() + if row: + db.delete(row) + db.commit() + return True + return False + + def create_chat(self, title, member_ids, creator_id): + if creator_id not in member_ids: + raise ValueError("CREATOR_NOT_MEMBER") + if len(member_ids) != len(set(member_ids)): + raise ValueError("DUPLICATE_MEMBER") + with SessionLocal() as db: + users = db.query(User.id).filter(User.id.in_(member_ids)).all() + if len(users) != len(member_ids): + raise ValueError("USER_NOT_FOUND") + chat = Chat(title=title) + chat.members = [ChatMember(user_id=mid) for mid in member_ids] + db.add(chat) + db.commit() + db.refresh(chat) + list(chat.members) + return chat + + def get_chat(self, chat_id): + with SessionLocal() as db: + return db.query(Chat).options(selectinload(Chat.members)).filter(Chat.id == chat_id).first() + + def get_user_chats(self, user_id, cursor=None, limit=50): + with SessionLocal() as db: + query = db.query(Chat).options(selectinload(Chat.members)).join(ChatMember).filter(ChatMember.user_id == user_id) + if cursor: + decoded = decode_cursor(cursor, "chats") + query = query.filter(or_(Chat.created_at > decoded[0], and_(Chat.created_at == decoded[0], Chat.id > decoded[1]))) + rows = query.order_by(Chat.created_at, Chat.id).limit(limit).all() + for row in rows: + list(row.members) + return rows + + def is_chat_member(self, chat_id, user_id): + with SessionLocal() as db: + return db.query(ChatMember.id).filter(ChatMember.chat_id == chat_id, ChatMember.user_id == user_id).first() is not None + + def add_message(self, chat_id, sender_id, text): + with SessionLocal() as db: + if not db.query(Chat.id).filter(Chat.id == chat_id).first(): + raise ValueError("CHAT_NOT_FOUND") + if not db.query(ChatMember.id).filter(ChatMember.chat_id == chat_id, ChatMember.user_id == sender_id).first(): + raise ValueError("NOT_MEMBER") + msg = Message(chat_id=chat_id, sender_id=sender_id, text=text) + db.add(msg) + db.commit() + db.refresh(msg) + return msg + + def add_message_idempotent(self, chat_id, sender_id, text, attachment_ids, key, request_hash): + """Claim a scoped key and create its message in the same DB transaction.""" + with SessionLocal() as db: + if not db.query(ChatMember.id).filter(ChatMember.chat_id == chat_id, + ChatMember.user_id == sender_id).first(): + raise ValueError("CHAT_NOT_FOUND") + if not text.strip() and not attachment_ids: + raise ValueError("EMPTY_MESSAGE") + if attachment_ids: + rows = db.query(Attachment).filter(Attachment.id.in_(attachment_ids), + Attachment.chat_id == chat_id).all() + if len(rows) != len(attachment_ids): + raise ValueError("ATTACHMENT_NOT_FOUND") + scope = dict(user_id=sender_id, method="POST", chat_path=f"/api/v1/chats/{chat_id}/messages", key=key) + claim = insert(MessageIdempotency).values(**scope, request_hash=request_hash, + message_id=None).on_conflict_do_nothing( + constraint="uq_idempotency_scope").returning(MessageIdempotency.id) + claimed = db.execute(claim).scalar_one_or_none() + if claimed is None: + record = db.execute(select(MessageIdempotency).filter_by(**scope)).scalar_one() + if record.request_hash != request_hash: + raise ValueError("IDEMPOTENCY_CONFLICT") + msg = db.get(Message, record.message_id) + if msg is None: + raise RuntimeError("Idempotency record has no message") + return msg + msg = Message(chat_id=chat_id, sender_id=sender_id, text=text, attachment_ids=attachment_ids) + db.add(msg) + db.flush() + db.execute(update(MessageIdempotency).where(MessageIdempotency.id == claimed).values(message_id=msg.id)) + db.commit() + db.refresh(msg) + return msg + + def get_messages(self, chat_id, cursor=None, limit=50): + with SessionLocal() as db: + query = db.query(Message).filter(Message.chat_id == chat_id) + if cursor: + created, message_id = decode_cursor(cursor, chat_id) + query = query.filter(or_(Message.created_at > created, + and_(Message.created_at == created, Message.id > message_id))) + return query.order_by(Message.created_at, Message.id).limit(limit).all() + + def save_attachment(self, attachment): + with SessionLocal() as db: + row = Attachment(**attachment) + db.add(row) + db.commit() + db.refresh(row) + return row + + def get_attachment(self, attachment_id, user_id): + with SessionLocal() as db: + return db.query(Attachment).join(ChatMember, ChatMember.chat_id == Attachment.chat_id).filter( + Attachment.id == attachment_id, ChatMember.user_id == user_id).first() + + +def encode_cursor(resource, created_at, item_id): + raw = json.dumps([resource, created_at.isoformat(), item_id], separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def decode_cursor(cursor, resource): + try: + data = json.loads(base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4))) + if data[0] != resource: + raise ValueError + return datetime.fromisoformat(data[1]), data[2] + except Exception: + raise ValueError("INVALID_CURSOR") + + +def cursor_for_chat(chat): + return encode_cursor("chats", chat.created_at, chat.id) + + +def cursor_for_message(message): + return encode_cursor(message.chat_id, message.created_at, message.id) + + +store = Store() diff --git a/student-tests/s3_unavailable.py b/student-tests/s3_unavailable.py new file mode 100644 index 00000000..591faa90 --- /dev/null +++ b/student-tests/s3_unavailable.py @@ -0,0 +1,43 @@ +"""Verify that S3 outages fail readiness/uploads cleanly and recover after restart.""" +import subprocess +import time +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, run # noqa: E402 + + +def main(): + args = arguments("MinIO outage and recovery checks") + # S3 health checks can spend the configured botocore connect timeout on an + # unreachable service before the API returns the expected JSON 503. + c = Client(args.base_url, args.ca_file, timeout=30) + alice, _bob, _eve, chat = c.bootstrap() + fixture = (ROOT / "tests/fixtures/sample.png").read_bytes() + subprocess.run(["docker", "compose", "--env-file", ".env", "stop", "minio"], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + # Nginx may hit its upstream timeout while both replicas are waiting + # for the S3 connect timeout, so 504 is also an expected outage result. + c.call("GET", "/health/ready", [503, 504]) + c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=outage.png", [503, 504], + actor=alice, raw=fixture, headers={"Content-Type": "image/png"}) + finally: + subprocess.run(["docker", "compose", "--env-file", ".env", "start", "minio"], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + deadline = time.monotonic() + 45 + while True: + try: + c.call("GET", "/health/ready", schema="Health") + break + except Exception: + if time.monotonic() >= deadline: + raise + time.sleep(0.5) + print("PASS: MinIO outage returned 503/504 for readiness/upload and recovered after service restart") + + +if __name__ == "__main__": + run(main)