diff --git a/.dockerignore b/.dockerignore index bc16e556..9c4c0b6c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,6 @@ __pycache__ node_modules evidence .DS_Store + +.superpowers +.pytest_cache diff --git a/.env.example b/.env.example index d99d26a2..6bcaec28 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,6 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# Created by scripts/bootstrap.sh; .env and secrets/ are never committed. SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 +# Compose supplies DATABASE_URL_FILE, CACHE_URL_FILE and S3 credentials *_FILE. +# S3_ENDPOINT=http://minio:9000, S3_REGION=us-east-1, S3_BUCKET=messenger-attachments. +# Contract limits:10MiB and40M pixels. Six uploads and one decoder per API. +# Cleanup uses grace300s, interval60s. Only Nginx publishes127.0.0.1:8080. diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..cd6219fc --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,26 @@ +name: Application acceptance +on: [push, pull_request] +permissions: + contents: read +jobs: + cluster: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: make up + - run: make check test test-student + - run: make test-fresh + - run: make test-cluster + - run: make test-attachments + - run: make uploads + - uses: actions/upload-artifact@v4 + if: always() + with: + name: lab4-evidence + path: evidence/ + - if: always() + run: docker compose down diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..8f8e4621 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.13.5-slim +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src +WORKDIR /app +COPY requirements.txt requirements-dev.txt requirements.lock ./ +RUN pip install --no-cache-dir -r requirements.lock -r requirements.txt && useradd --uid 10001 --create-home messenger +COPY src/ src/ +COPY migrations/ migrations/ +COPY student-tests/ student-tests/ +COPY scripts/ scripts/ +COPY tests/ tests/ +COPY contracts/ contracts/ +COPY lab.json Makefile README.md COURSE.md CONTRIBUTING.md REPORT.md ./ +ARG APP_RELEASE=lab4-v1 +ENV APP_RELEASE=${APP_RELEASE} +LABEL org.opencontainers.image.version=${APP_RELEASE} +USER messenger +EXPOSE 8080 +CMD ["uvicorn", "messenger.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5", "--no-access-log"] diff --git a/Makefile b/Makefile index 5d86d171..7688f57c 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,42 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: bootstrap up down test-student test-cluster rolling +bootstrap: + ./scripts/bootstrap.sh +up: bootstrap + docker compose build api minio minio-init + docker compose up -d --wait --wait-timeout 120 + $(PYTHON) scripts/cluster.py wait +down: + docker compose down +test-student: + $(PYTHON) scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache +test-cluster: + $(PYTHON) scripts/verify_cluster.py +rolling: + $(PYTHON) scripts/cluster.py rolling + +.PHONY: test-fresh load-evidence verify-persistence verify-failover +test-fresh: + ./scripts/check_fresh_migrations.sh +load-evidence: + $(PYTHON) scripts/capture_load.py +verify-persistence: + $(PYTHON) scripts/verify_cluster.py persistence +verify-failover: + $(PYTHON) scripts/verify_cluster.py public-failover + $(PYTHON) scripts/verify_cluster.py failover + +.PHONY: validate-container +validate-container: + $(PYTHON) scripts/cluster.py exec python scripts/validate_openapi.py + +.PHONY: test-attachments uploads cleanup-dry-run +test-attachments: + $(PYTHON) scripts/verify_attachments.py +uploads: + $(PYTHON) scripts/capture_uploads.py +cleanup-dry-run: + docker compose run --rm --no-deps cleanup python -m messenger.cleanup --dry-run diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..75404f00 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,149 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Лукьянов К.Р. БПИ2303 +- Целевая оценка; выбранный трек 5, если нужен: **4** (трек 5 не заявлен) - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники, библиотеки и помощь ИИ: FastAPI, Pydantic v2, Uvicorn, Psycopg 3, redis-py, argon2-cffi, boto3/botocore, Pillow, pytest; документация [S3 uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html), [Pillow Image](https://pillow.readthedocs.io/en/stable/reference/Image.html), [Boto3 configuration](https://docs.aws.amazon.com/boto3/latest/guide/configuration.html), [MinIO IAM](https://github.com/minio/minio/blob/master/docs/multi-user/README.md), [Nginx upstream](https://nginx.org/en/docs/http/ngx_http_upstream_module.html). Реализация выполнена совместно мной и Codex: часть кода, тестов и отчёта написана с помощью Codex, результат проверен запуском тестов, описанных ниже. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker с Compose v2, Python ≥ 3.9 на хосте для стандартных тестов/оркестрации, Make, свободный localhost:8080. Проверено на Docker Desktop 29.8.2, macOS/Apple M4 Pro. Go и Python-зависимости приложения устанавливаются внутри сборочных контейнеров. Для первой сборки нужен доступ к GitHub и Go/Python module repositories. + +```sh +git clone && cd <каталог-fork-лабораторной-4> +make up # secrets → сборка API/MinIO/mc → migrate/init → api×2/cleanup/Nginx +curl -fsS http://localhost:8080/health/ready +docker compose ps --all # API×2/PG/Redis/MinIO/cleanup/LB работают; migrate и minio-init — Exited(0) + +make check +make validate-container +make test # публичный smoke lab4 через Nginx +make test-student # 21 собственный интеграционный тест +make test-fresh # миграции 001–003, новый bucket и тесты на отдельных пустых volumes +make test-attachments # private bucket, upload→отказ API→download, обрыв, crash+cleanup, отказ S3 +make uploads # 10 одновременных загрузок с измерением пика RAM +make test-cluster # сохранённые проверки общей сессии/idempotency/restart/failover/SIGTERM/rolling +``` + +`make up` выполняет bootstrap, `docker compose build api minio minio-init`, затем `up --wait` и ожидание двух API-alias через LB. При повторной сборке Go/Python-слои используются из кэша. Сценарии отказов и замер памяти запускать по одному: они останавливают/пересоздают контейнеры. `make uploads` пересоздаёт только API, чтобы сбросить kernel memory high-water marks; volumes и данные сохраняются. + +**Почему MinIO собирается из исходников:** проверенные release-образы в Docker Hub/Quay недоступны, официальный архив бинарников ответил 410. В `infra/minio/Dockerfile` собираются официальные теги и проверяется точный commit: MinIO `RELEASE.2025-04-22T22-12-26Z` / `0d7408fc9969caf07de6a8c3a84f9fbb10a6739e`, mc `RELEASE.2025-04-16T18-13-26Z` / `b00526b153a31b36767991a4f5ce2cced435ee8e`. Builder — `golang:1.24.2-bookworm`, зависимости сверяются по upstream go.sum. Runtime-образы локальные `mtusi-lab4-minio:2025-04-22` и `mtusi-lab4-mc:2025-04-16`; скачивать готовый сервер преподавателя не требуется. + +`bootstrap.sh` однократно создаёт пароль БД, root-credentials MinIO и отдельные API S3 credentials в `secrets/`; повтор их не меняет. Каталог 0700 на хосте, файлы доступны нужному uid через Compose secrets. Root S3 secrets получает только MinIO и minio-init; API/cleanup получают ограниченного пользователя. `.env`, secrets и volumes не входят в Git/build context. Учебные данные и credentials lab3 не копировались. + +Безопасная остановка: `make down`, именованные volumes `postgres_data`, `redis_data`, `minio_data` сохраняются. Намеренное удаление всех учебных данных: `docker compose down --volumes`. Restart hook volumes не удаляет. `make test-fresh` удаляет только созданные им временные volumes отдельного проекта. После обычных тестов синтетические пользователи/сообщения/вложения остаются — delete не входит в обязательный API. + +Отдельные команды: + +```sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +# После отдельного публичного failover восстановить остановленную реплику: +docker compose up -d --no-recreate --wait --wait-timeout 90 +python3 scripts/cluster.py wait + +make cleanup-dry-run +# Явная безопасная очистка с обычным grace 300s: +docker compose run --rm --no-deps cleanup python -m messenger.cleanup --apply +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +клиент → Nginx → API ×2 ──→ PostgreSQL: пользователи/чаты/сообщения/сессии/metadata/связи + ├─→ Redis: кэш публичного профиля + └─→ MinIO: оригинальные байты, приватный bucket + +migrate → PostgreSQL minio-init → bucket/user/policy +cleanup → S3 listing + PostgreSQL recheck → удаление только старых неучтённых объектов +``` + +- **Перенос из lab3.** Сохранены Nginx, два экземпляра одного service api, общая PostgreSQL, Argon2id/Basic→Bearer, Redis cache-aside профиля, keyset-пагинация, X-Instance-Id и транзакционная идемпотентность сообщений. Сохранены публичные tests/contracts/README текущего skeleton. Служебные документы находятся отдельно в `lab4/docs/`. +- **Схема** (`003_attachments.sql`). `attachments`: UUID PK, chat_id/owner_id FK, составной FK на членство, filename, MIME, размер 1–10 MiB, SHA-256, уникальный object_key, created_at/updated_at. `message_attachments`: FK на сообщение/вложение, position 0–9, уникальны пара и позиция. Байты не хранятся в PostgreSQL. Проверка same-chat перед INSERT связи даёт 404 для чужого/неизвестного attachment_id. Любой участник может использовать доступное вложение своего чата. +- **Совместимость сообщения.** В ответах добавлен attachment_ids; в запросе по умолчанию []. Пустой/пробельный text допустим только с вложением. Список не содержит дублей, до 10 UUID, порядок сохраняется. Отсутствие списка и [] дают одинаковый fingerprint; пустой список не включается в fingerprint, поэтому сохраняется hash прежних text-only запросов lab3. Replay возвращает прежний Message 201, другой валидный body 409, доступ и сессия перепроверяются. +- **Пути и ACL.** Object key — `originals/<случайный UUID>`, не filename и не публичный URL. Имя — только отображаемое поле: 1–255 Unicode code points, не пустое, без slash/backslash/NUL/control. Сначала Bearer и членство, затем загрузка; metadata/content доступны только участникам, посторонний/неизвестный объект 404. Публичный JSON не раскрывает object key, bucket, endpoint или credentials. +- **Получение файла.** POST принимает сырой PNG/JPEG, не multipart. Content-Length проверяется заранее, но фактически полученные bytes тоже считаются. Тело читается через Request.stream в TemporaryFile на диске, SHA-256 считается по ходу. Лимит 10 MiB и абсолютное ожидание тела 30 с; слот загрузки ожидается до 1 с. Файл удаляется/закрывается в finally, постоянной локальной копии оригинала у API нет. +- **Проверка изображения.** Pillow определяет формат и размеры до load; MIME должен совпадать. >40 млн пикселей даёт 413 до полной распаковки. verify проверяет структуру, повторный open/load — декодирование. Повреждённый/неподдерживаемый файл 415, слишком большой 413, опасное имя 400. Текстовые PNG chunks ограничены 1 MiB. На API один decoder и не более 6 одновременно обрабатываемых upload: несколько больших распаковок не размножают расход RAM. Оригинальные байты не перекодируются; crop/thumbnail/очереди относятся к следующим работам. +- **Буферы Nginx.** На upload-route client_max_body_size 11 MiB, чтобы 10 MiB+1 дошёл до API и получил 413; client_body_buffer_size 64 KiB, остальное буферизуется на диск. `proxy_request_buffering on`, максимум 16 параллельных загрузок на LB, inactivity timeout тела 10 с, upload read/send timeout 30 с. Это сознательное дисковое буферизование, не обещание мгновенного приёма произвольно медленного клиента. +- **201 и согласованность.** После валидации API начинает PG-транзакцию и берёт advisory lock от object_key. Затем выполняются S3 PUT и INSERT metadata; 201 возвращается только после успешного COMMIT. У S3 и PostgreSQL нет общей атомарной транзакции. Подтверждённые metadata доступны всем API, оригинал находится в общем MinIO. +- **Компенсация неизвестного результата.** Ошибка PUT/SQL/COMMIT вызывает новую проверку под тем же lock. Объект удаляется только если БД точно подтверждает отсутствие metadata. При неизвестном COMMIT или недоступной БД слепого DeleteObject нет: уже учтённый файл должен остаться, неучтённый подберёт janitor. Обрыв неполного клиентского тела не начинает S3 PUT; аварийное завершение после PUT может оставить orphan. +- **Автоматическая очистка.** Отдельный cleanup-service каждые 60 с листает только originals/* страницами до 1000 объектов. Grace по умолчанию 300 с, минимум 60 с. Для старого кандидата pg_try_advisory_xact_lock и свежий SELECT повторно проверяют metadata; учтённый/свежий/активно заблокированный объект пропускается. При ошибке БД удаления не происходит. Dry-run не удаляет, повтор apply идемпотентен. Здесь реализована компенсация/orphan cleanup уровня 4, не полный исследовательский reconciler из трека 5 и не восстановление missing bytes. +- **Скачивание.** GET content возвращает StreamingBody S3 кусками 64 KiB через авторизованный API, без redirect. До 8 открытых S3-stream на API; close/release выполняется в finally ответа, включая ошибку отправки headers и disconnect. Content-Length сверяется с metadata. Variant в lab4 разрешён только original; прочие значения 400. Истинный MIME и RFC5987 filename передаются в headers. +- **MinIO и права.** minio-init идемпотентно создаёт bucket, запрещает anonymous, создаёт отдельного API user и policy. Get/Put/Delete разрешены только originals/* этого bucket; ListBucket — только для этого prefix; GetBucketLocation — для bucket. CreateBucket, чужие bucket/prefix и admin-права не выдаются. Console слушает 127.0.0.1:9001 внутри MinIO, не опубликована и недоступна из API-сети. Приватный bucket не заменяет ACL приложения. +- **Здоровье и таймауты.** Readiness требует PostgreSQL, совместимую схему и доступный S3 bucket с API credentials; Redis — необязательный кэш. Boto3 connect 1 с/read 3 с, одна попытка, пул/stream gate 8. PG-пул 8 на API, ожидание 2 с, statement 3 с/lock 2 с. S3 outage даёт 503 API либо 502/503/504 LB после исключения backend. Nginx OSS использует Docker DNS resolve и пассивные max_fails 1/fail_timeout 2 с; активный health_check Plus не используется. SIGTERM/rolling сохраняют схему lab3. + +Роли, ресурсы и запуск: + +| Service | Версия / ресурсы | Данные, сеть и готовность | +| --- | --- | --- | +| api×2 | `mtusi-lab4-api:lab4`; каждая 1 CPU/512 MiB, uid10001 | front+data, без host ports; HTTP ready, только ограниченные S3 secrets | +| lb | `nginx:1.28.0-alpine`; 0.5 CPU/128 MiB | только front; единственный public endpoint 127.0.0.1:8080 | +| postgres | `postgres:17.5-bookworm`; 1 CPU/512 MiB | internal data; postgres_data, pg_isready | +| redis | `redis:7.4.4-bookworm`; 0.25 CPU/128 MiB | internal data; redis_data, AOF, ping | +| minio | официальный release 2025-04-22, сборка из pinned source; 1 CPU/512 MiB, uid10002 | internal data; minio_data; HTTP /minio/health/ready; root secrets | +| minio-init | официальный mc release 2025-04-16, сборка; 0.25 CPU/128 MiB | root+API secrets, policy read-only, Exited(0) | +| migrate | образ API; 0.5 CPU/256 MiB | только DB secret, версии/lock/checksum, Exited(0) до API | +| cleanup | образ API; 0.25 CPU/192 MiB | DB+ограниченные S3 secrets, только data, цикл 60 с | + +Python 3.13.5, библиотеки и транзитивные зависимости закреплены в requirements.lock. В .env реально используется SESSION_TTL_SECONDS; S3_ENDPOINT/REGION/BUCKET и *_FILE задаются Compose. Контрактные пределы 10 MiB/40 Mp не увеличиваются через переменные. Отдельные migrate и minio-init завершаются до API. Dockerfile приложения не запускает миграции каждой репликой. ## Доказательства критериев +Основные эксперименты — 2026-11-02; контрольный запуск — 2026-11-09. Среда: macOS/Apple M4 Pro, 24 GiB RAM; Docker VM 12 CPU и около 7.75 GiB. Ветка `solution/bpi2303-lukyanov`. Измерения относятся к текущему учебному стенду, пароль/токен в evidence не сохранялись. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Контракт и публичный API (3) | make check/validate-container/test | PASS: lab4 public API scenario, checksum, ACL и сообщения с ids | evidence/check.txt, openapi.txt, smoke.txt | +| Метаданные в PG, bytes в приватном S3 (3) | схема 003, security test, download | готовые originals в bucket, API не раскрывает key; anonymous 403 | migrations/003_attachments.sql, evidence/s3-security.json | +| PNG/JPEG и исходные bytes (3) | собственные тесты + smoke | 201 ready, SHA-256/размер/bytes совпадают, image-only сообщение валидно | evidence/student-tests.txt, smoke.txt | +| Byte/pixel/MIME/filename ограничения (3) | тесты повреждений, >10 MiB, chunked, >40 Mp header | 400/413/415; pixel limit до load, пользовательская ошибка не 500 | evidence/student-tests.txt | +| Upload на одной, download на другой (3) | verify_attachments.py failover | остановлен точно upload-instance; другой вернул тот же hash/bytes | evidence/attachment-failover.json | +| Полный restart с S3 volume (3) | verify_cluster.py persistence | прежняя сессия, metadata и оригинал доступны; реальные процессы перезапущены | evidence/persistence.txt, persistence.json | +| Прежняя балансировка/идемпотентность (3) | test-cluster, собственные регрессии | concurrent replay, failover/SIGTERM/rolling проходят | evidence/concurrent-idempotency.json, public-failover.txt, sequential-failover.json, graceful-shutdown.json, rolling-update.json | +| Все отрицательные случаи файлов/чужих ids (4) | make test-student | 21 тест PASS, включая MIME, corrupt/oversize, чужой attachment и другой чат | evidence/student-tests.txt | +| Компенсация после PUT/DB failure (4) | SQL trigger перед metadata INSERT, затем реальный SIGKILL | SQL-сбой 503 компенсирован; orphan после kill обнаружен и удалён после grace | evidence/student-tests.txt, orphan-cleanup.json | +| Безопасная очистка (4) | dry-run/apply/repeat; fresh/tracked/locked | dry-run: 0 удалений; apply удалил orphan; повтор 0; свежий/используемый/locked сохранены | evidence/orphan-cleanup.json | +| Потоковая обработка и RAM при ≥10 загрузках (4) | make uploads | 10/10 одновременно, 2 API, 0 ошибок/OOM; пик 205.824/214.730 MiB при 512 MiB | evidence/concurrent-uploads.json | +| Bucket bootstrap идемпотентен, IAM ограничен (4) | повтор minio-init; прямые S3-запросы | objects/metadata не изменились; anonymous 403, CreateBucket 403, другой prefix: 403 | evidence/s3-security.json | +| Консоль не открыта (4) | socket из API к minio:9001; infrastructure snapshot | соединение не установлено, root secret в API не смонтирован, host ports только у LB | evidence/s3-security.json, infrastructure.json | +| Миграции/инициализация с нуля | make test-fresh | schema 001–003 и private bucket на новых volumes, тесты PASS | evidence/fresh-migrations.txt | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Pytest также выводит предупреждение старой закреплённой botocore об utcnow в Python 3.13 и диагностическое предупреждение Pillow в тесте pixel-bomb; проверки результата 413 и все assertions проходят. Публичные проверки и контракт не ослаблялись. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Исследовательский трек 5 не выполнялся. Файлы синтетические. Обрыв тела и сбой между PUT/COMMIT проверяются раздельно: одного успешного API-запроса недостаточно для доказательства компенсации. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потери | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Upload → stop обслужившей API → download | файл читается через другую API | тот же SHA-256 и bytes, другой alias | 1.988 с от старта действия / потерь нет | +| Обрыв HTTP upload | неполное тело не создаёт объект/metadata | из 500000 объявленных передано 108 bytes, TCP закрыт; counts не изменились | orphan нет | +| Ошибка INSERT metadata после S3 PUT | rollback PG и компенсация S3 | HTTP 503, нет строки и нового объекта | подтверждения не было | +| SIGKILL после PUT, до COMMIT | неподтверждённый объект может остаться и будет очищен | наблюдался exit 137; объект есть, metadata нет; после 61 с реального ожидания dry-run показал кандидата, apply удалил его | orphan устранён, 201 не выдан | +| Repeat cleanup, fresh/linked/locked | безопасные повторы | повтор не удаляет объекты; свежий и используемый сохранены, объект под lock сохранён до unlock | потерь используемых данных нет | +| MinIO недоступен | ограниченная ошибка без ложного 201 | upload 503 за 0.2487 с; после старта прежний оригинал читается | тот же токен, bytes сохранены | +| Полный restart API/PG/Redis/MinIO/cleanup/Nginx | сессия и оригинал переживают restart | public persistence PASS | именованные volumes сохранены | + +**Методика crash/cleanup:** временный BEFORE INSERT trigger задерживает ровно одно имя файла через pg_sleep(2). После наблюдения INSERT в pg_stat_activity и появления объекта в S3 API получает SIGKILL. Metadata не зафиксирована, 201 не отправлен. Контейнер восстановлен, trigger удалён. После настоящего 61-секундного ожидания применяется grace 60 с. Дополнительно проверены старый объект под advisory lock, свежий объект и старое вложение, связанное с сообщением. После снятия lock его тестовый orphan также удалён. В обычном запуске grace 300 с, цикл 60 с; при недоступной БД уборка откладывается. + +**Память при десяти загрузках:** воспроизводимый RGB PNG 3000×1000, 9002318 bytes, zlib level 0, строки из SHAKE-256. Один неизменяемый payload используется десятью клиентскими потоками с barrier. Все десять HTTP-интервалов перекрывались 0.345 с, завершились 201, обе API наблюдались, ошибок 0. Время группы 1.563 с. Один оригинал скачан другим участником и побайтно сверен. + +Перед опытом API пересозданы, чтобы memory.peak и VmHWM относились к этому запуску. Peak включает startup/bootstrap/загрузки/контрольное скачивание. Kernel high-water mark не пропускает короткие пики между samples: + +| API-процесс | Cgroup memory peak | PID 1 VmHWM | Лимит | OOM | +| --- | --- | --- | --- | --- | +| Первый | 205.824 MiB | 182.871 MiB | 512 MiB | 0 | +| Второй | 214.730 MiB | 183.293 MiB | 512 MiB | 0 | + +Cgroup включает cache и вспомогательные процессы, VmHWM — resident peak API. Профиль, версии, limits, baseline/final, интервалы запросов и checksum — в evidence/concurrent-uploads.json. Опыт показывает расход памяти этого профиля, а не максимальный throughput или универсальный RSS для всех допустимых JPEG/PNG. Сценарии lab3 под текстовой нагрузкой сохранены отдельно; их числа не заменяют эксперимент с файлами. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host, одна PostgreSQL и один MinIO: отказ узла/диска и HA хранилищ не покрываются. Named volume не заменяет backup. Потерянные bytes из missing object janitor восстановить не может. +- S3 и PostgreSQL неатомарны. Между PUT и COMMIT есть окно orphan; компенсация и grace-cleanup закрывают его после восстановления зависимостей. При недоступной БД удаление запрещено; время уборки не является SLA. +- Janitor удаляет только неучтённые старые объекты. Учтённое, но никуда не прикреплённое вложение сохраняется: в обязательном API нет delete/retention для таких файлов. +- Readiness считает S3 критичной зависимостью. При S3 outage Nginx может исключить обе API, и text-only запросы тоже станут недоступны, хотя PG жив. Внутренний live процесса остаётся 200. +- Таймауты SDK/Nginx относятся к отдельным операциям I/O. После начала HTTP download нельзя заменить частичное 200 структурированным 503; клиент проверяет размер/checksum. +- Нагрузочный замер: десять одновременных загрузок по 9 MB, две API, один прогон. Исследования максимального RPS, presigned upload, thumbnail/crop и фоновой обработки lab5 не проводились. +- MinIO собран из закреплённого исторического релиза; стенд локальный и учебный. HTTP/Basic только на localhost с синтетическими данными; TLS, rotation, secret manager и полное разделение DB privileges оставлены следующим работам. +- Между страницами нет общего snapshot. Сохраняются оговорённые в lab3 ограничения часов и конкурентных вставок. Breaking DB migrations потребуют отдельной стратегии. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..62d33dcb --- /dev/null +++ b/compose.yaml @@ -0,0 +1,150 @@ +name: mtusi-lab4 +services: + postgres: + image: postgres:17.5-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + networks: [data] + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 512m + cpus: 1.0 + redis: + image: redis:7.4.4-bookworm + command: [redis-server, --appendonly, 'yes', --maxmemory, 64mb, --maxmemory-policy, allkeys-lru] + volumes: [redis_data:/data] + networks: [data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 128m + cpus: 0.25 + migrate: + image: mtusi-lab4-api:lab4 + command: [python, -m, messenger.migrate] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: {condition: service_healthy} + restart: 'no' + mem_limit: 256m + cpus: 0.5 + api: + image: mtusi-lab4-api:lab4 + build: . + scale: 2 + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + POOL_MAX_SIZE: '8' + S3_ENDPOINT: http://minio:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + networks: [front, data] + depends_on: + migrate: {condition: service_completed_successfully} + redis: {condition: service_healthy} + minio-init: {condition: service_completed_successfully} + stop_grace_period: 10s + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"] + interval: 2s + timeout: 5s + retries: 30 + mem_limit: 512m + cpus: 1.0 + minio: + image: mtusi-lab4-minio:2025-04-22 + build: + context: . + dockerfile: infra/minio/Dockerfile + target: minio + secrets: [minio_root_user, minio_root_password] + volumes: [minio_data:/data] + networks: [data] + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:9000/minio/health/ready',timeout=2)"] + interval: 2s + timeout: 3s + retries: 30 + stop_grace_period: 10s + mem_limit: 512m + cpus: 1.0 + minio-init: + image: mtusi-lab4-mc:2025-04-16 + build: + context: . + dockerfile: infra/minio/Dockerfile + target: mc + secrets: [minio_root_user, minio_root_password, s3_access_key_id, s3_secret_access_key] + volumes: + - ./infra/minio/bootstrap.sh:/config/bootstrap.sh:ro + - ./infra/minio/api-policy.json:/config/api-policy.json:ro + networks: [data] + depends_on: + minio: {condition: service_healthy} + restart: 'no' + mem_limit: 128m + cpus: 0.25 + cleanup: + image: mtusi-lab4-api:lab4 + command: [python, -m, messenger.cleanup, --loop, --apply, --grace-seconds, '300', --interval, '60'] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://minio:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + migrate: {condition: service_completed_successfully} + minio-init: {condition: service_completed_successfully} + mem_limit: 192m + cpus: 0.25 + lb: + image: nginx:1.28.0-alpine + volumes: ['./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro'] + networks: [front] + ports: ['127.0.0.1:8080:8080'] + depends_on: + api: {condition: service_healthy} + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready] + interval: 1s + timeout: 8s + retries: 30 + mem_limit: 128m + cpus: 0.5 +networks: + front: + data: + internal: true +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + minio_root_user: {file: ./secrets/minio_root_user} + minio_root_password: {file: ./secrets/minio_root_password} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} +volumes: + postgres_data: + redis_data: + minio_data: diff --git a/evidence/attachment-failover.json b/evidence/attachment-failover.json new file mode 100644 index 00000000..b965b703 --- /dev/null +++ b/evidence/attachment-failover.json @@ -0,0 +1,18 @@ +{ + "result": "PASS", + "stopped": { + "container": "01ac5da3549e234a658a6202029fec3661951caf9cd6230f71536579465b51df", + "name": "mtusi-lab4-api-1", + "instance": "68057e4f671d4bc4beebe6446a39f551", + "release": "lab4-v1", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "stopped_at_utc": "2026-11-02T09:10:52.334643+00:00", + "stop_seconds": 0.415, + "exit_code": 0, + "left_stopped": true + }, + "download_instance": "4bd9e299c91d4af99e1b94bab75f8f3d", + "same_bytes": true, + "same_sha256": true, + "seconds_from_action_start": 1.988 +} diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..9a553306 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/concurrent-idempotency.json b/evidence/concurrent-idempotency.json new file mode 100644 index 00000000..21d0ed68 --- /dev/null +++ b/evidence/concurrent-idempotency.json @@ -0,0 +1,11 @@ +{ + "result": "PASS", + "parallel_requests": 8, + "observed_instances": [ + "3ca208a978e441ccba13625143527d66", + "de89643f69534e43bc1111510e0a993e" + ], + "responses_201": 8, + "messages_created": 1, + "different_body_status": 409 +} diff --git a/evidence/concurrent-uploads.json b/evidence/concurrent-uploads.json new file mode 100644 index 00000000..353c9b09 --- /dev/null +++ b/evidence/concurrent-uploads.json @@ -0,0 +1,612 @@ +{ + "captured_at_utc": "2026-11-02T09:15:37.434138+00:00", + "result": "PASS", + "context": { + "git_commit": "ec540a1a725862a8619fcb758daec4ad197f0dd5", + "tracked_worktree_dirty": true, + "host_python": "3.13.7", + "host_os": "macOS-27.2-arm64-arm-64bit-Mach-O", + "docker_version": "29.8.2", + "compose_version": "5.6.0", + "contract_version": "1.4.0", + "endpoint": { + "scheme": "http", + "host": "localhost", + "port": 8080 + } + }, + "input": { + "format": "PNG", + "mode": "RGB", + "width": 3000, + "height": 1000, + "size_bytes": 9002318, + "sha256": "9bb84f9e461995a5e031a0678da59c6dea8d963a157e590a9798d9030437c3fe", + "generator": "SHAKE-256 lab4-concurrent-upload-v1 row seeds; filter 0; zlib level 0", + "same_immutable_bytes_for_all_requests": true + }, + "concurrency": 10, + "reset": { + "command": "docker compose up -d --no-deps --force-recreate api", + "purpose": "Reset API cgroup memory.peak and PID 1 VmHWM; retain all volumes" + }, + "peak_scope": "Fresh API startup, bootstrap, ten uploads and representative download; no peak sampling gaps", + "cluster_before": [ + { + "service": "api", + "container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c", + "name": "mtusi-lab4-api-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:15:38.382323712Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data", + "mtusi-lab4_front" + ], + "http_status": 200, + "instance": "3bd3bbc5d37c4b5a97dc188076f363dd", + "release": "lab4-v1" + }, + { + "service": "api", + "container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72", + "name": "mtusi-lab4-api-2", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:15:38.308471337Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data", + "mtusi-lab4_front" + ], + "http_status": 200, + "instance": "de89643f69534e43bc1111510e0a993e", + "release": "lab4-v1" + }, + { + "service": "cleanup", + "container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6", + "name": "mtusi-lab4-cleanup-1", + "status": "running", + "health": null, + "started_at": "2026-11-02T09:10:38.357370005Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 201326592, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "lb", + "container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71", + "name": "mtusi-lab4-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:03:39.330794254Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab4_front" + ] + }, + { + "service": "migrate", + "container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2", + "name": "mtusi-lab4-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-02T09:10:37.212308463Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "minio", + "container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36", + "name": "mtusi-lab4-minio-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:12:16.586994384Z", + "image": "mtusi-lab4-minio:2025-04-22", + "image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_minio_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "minio-init", + "container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641", + "name": "mtusi-lab4-minio-init-1", + "status": "exited", + "health": null, + "started_at": "2026-11-02T09:10:37.288107921Z", + "image": "mtusi-lab4-mc:2025-04-16", + "image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "postgres", + "container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2", + "name": "mtusi-lab4-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:03:32.508777334Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_postgres_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "redis", + "container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a", + "name": "mtusi-lab4-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:03:32.445843167Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_redis_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + } + ], + "baseline": { + "3bd3bbc5d37c4b5a97dc188076f363dd": { + "source": "cgroup-v2", + "cgroup_peak_bytes": 96063488, + "cgroup_current_bytes": 87937024, + "pid1_vmhwm_bytes": 96518144, + "pid1_vmrss_bytes": 96518144, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0, + "sock_throttled": 0 + } + }, + "de89643f69534e43bc1111510e0a993e": { + "source": "cgroup-v2", + "cgroup_peak_bytes": 104452096, + "cgroup_current_bytes": 87433216, + "pid1_vmhwm_bytes": 96370688, + "pid1_vmrss_bytes": 96370688, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0, + "sock_throttled": 0 + } + } + }, + "memory_limits": [ + { + "instance": "3bd3bbc5d37c4b5a97dc188076f363dd", + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + { + "instance": "de89643f69534e43bc1111510e0a993e", + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + } + ], + "requests": [ + { + "index": 0, + "start_seconds": 0.9074667920067441, + "status": 201, + "instance": "3bd3bbc5d37c4b5a97dc188076f363dd", + "attachment_id": "14a5f6e5-0973-4b7f-98e2-81a9b7ba6ccb", + "metadata_verified": true, + "end_seconds": 1.4336796249990584 + }, + { + "index": 1, + "start_seconds": 0.9073489169968525, + "status": 201, + "instance": "de89643f69534e43bc1111510e0a993e", + "attachment_id": "79f74d1b-624f-497e-b80d-e2605dc22427", + "metadata_verified": true, + "end_seconds": 1.5235847080039093 + }, + { + "index": 2, + "start_seconds": 0.9075777499965625, + "status": 201, + "instance": "3bd3bbc5d37c4b5a97dc188076f363dd", + "attachment_id": "c5e24f93-8f8b-49fc-9e71-13161f08be4b", + "metadata_verified": true, + "end_seconds": 1.2526268750079907 + }, + { + "index": 3, + "start_seconds": 0.907239958003629, + "status": 201, + "instance": "de89643f69534e43bc1111510e0a993e", + "attachment_id": "480d346b-f983-45a3-9a78-05064f27ef06", + "metadata_verified": true, + "end_seconds": 1.352467999997316 + }, + { + "index": 4, + "start_seconds": 0.9076472919987282, + "status": 201, + "instance": "3bd3bbc5d37c4b5a97dc188076f363dd", + "attachment_id": "9e96aac9-21a4-4b27-b600-881e440e17b7", + "metadata_verified": true, + "end_seconds": 1.5511514579993673 + }, + { + "index": 5, + "start_seconds": 0.9077015420043608, + "status": 201, + "instance": "3bd3bbc5d37c4b5a97dc188076f363dd", + "attachment_id": "85dde29d-dccf-43a8-b634-4fb380ce555f", + "metadata_verified": true, + "end_seconds": 1.5152983750012936 + }, + { + "index": 6, + "start_seconds": 0.9077392080071149, + "status": 201, + "instance": "de89643f69534e43bc1111510e0a993e", + "attachment_id": "63ab944f-fd09-465e-bd9a-ad033cf79477", + "metadata_verified": true, + "end_seconds": 1.5623937500058673 + }, + { + "index": 7, + "start_seconds": 0.9075430829980178, + "status": 201, + "instance": "de89643f69534e43bc1111510e0a993e", + "attachment_id": "4b5e0c15-4a6c-47a4-9741-74cd6ac3b3db", + "metadata_verified": true, + "end_seconds": 1.5540448750107316 + }, + { + "index": 8, + "start_seconds": 0.9074074170057429, + "status": 201, + "instance": "de89643f69534e43bc1111510e0a993e", + "attachment_id": "38272fe7-3a6a-4064-9664-f7346a36431a", + "metadata_verified": true, + "end_seconds": 1.5352119580056751 + }, + { + "index": 9, + "start_seconds": 0.9075136250030482, + "status": 201, + "instance": "3bd3bbc5d37c4b5a97dc188076f363dd", + "attachment_id": "5c124808-e996-4e34-bfe6-16f20fbe6895", + "metadata_verified": true, + "end_seconds": 1.5333833750046324 + } + ], + "elapsed_seconds": 1.562777917002677, + "peak_active_requests": 10, + "all_ten_overlap_seconds": 0.3448876670008758, + "all_ten_overlapping": true, + "successful_count": 10, + "error_count": 0, + "observed_instances": [ + "3bd3bbc5d37c4b5a97dc188076f363dd", + "de89643f69534e43bc1111510e0a993e" + ], + "representative_download": { + "verified_as": "bob (chat member)", + "matches_original_bytes": true, + "size_bytes": 9002318, + "sha256": "9bb84f9e461995a5e031a0678da59c6dea8d963a157e590a9798d9030437c3fe" + }, + "final": { + "3bd3bbc5d37c4b5a97dc188076f363dd": { + "source": "cgroup-v2", + "cgroup_peak_bytes": 215822336, + "cgroup_current_bytes": 183332864, + "pid1_vmhwm_bytes": 191754240, + "pid1_vmrss_bytes": 191754240, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0, + "sock_throttled": 0 + } + }, + "de89643f69534e43bc1111510e0a993e": { + "source": "cgroup-v2", + "cgroup_peak_bytes": 225161216, + "cgroup_current_bytes": 182919168, + "pid1_vmhwm_bytes": 192196608, + "pid1_vmrss_bytes": 192196608, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0, + "sock_throttled": 0 + } + } + }, + "cluster_after": [ + { + "service": "api", + "container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c", + "name": "mtusi-lab4-api-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:15:38.382323712Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data", + "mtusi-lab4_front" + ], + "http_status": 200, + "instance": "3bd3bbc5d37c4b5a97dc188076f363dd", + "release": "lab4-v1" + }, + { + "service": "api", + "container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72", + "name": "mtusi-lab4-api-2", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:15:38.308471337Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data", + "mtusi-lab4_front" + ], + "http_status": 200, + "instance": "de89643f69534e43bc1111510e0a993e", + "release": "lab4-v1" + }, + { + "service": "cleanup", + "container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6", + "name": "mtusi-lab4-cleanup-1", + "status": "running", + "health": null, + "started_at": "2026-11-02T09:10:38.357370005Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 201326592, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "lb", + "container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71", + "name": "mtusi-lab4-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:03:39.330794254Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab4_front" + ] + }, + { + "service": "migrate", + "container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2", + "name": "mtusi-lab4-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-02T09:10:37.212308463Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "minio", + "container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36", + "name": "mtusi-lab4-minio-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:12:16.586994384Z", + "image": "mtusi-lab4-minio:2025-04-22", + "image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_minio_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "minio-init", + "container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641", + "name": "mtusi-lab4-minio-init-1", + "status": "exited", + "health": null, + "started_at": "2026-11-02T09:10:37.288107921Z", + "image": "mtusi-lab4-mc:2025-04-16", + "image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "postgres", + "container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2", + "name": "mtusi-lab4-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:03:32.508777334Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_postgres_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "redis", + "container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a", + "name": "mtusi-lab4-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:03:32.445843167Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_redis_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + } + ], + "oom_killed": { + "mtusi-lab4-api-1": false, + "mtusi-lab4-api-2": false + }, + "per_node_peaks": { + "3bd3bbc5d37c4b5a97dc188076f363dd": { + "source": "cgroup-v2", + "cgroup_peak_bytes": 215822336, + "cgroup_current_bytes": 183332864, + "pid1_vmhwm_bytes": 191754240, + "pid1_vmrss_bytes": 191754240, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0, + "sock_throttled": 0 + } + }, + "de89643f69534e43bc1111510e0a993e": { + "source": "cgroup-v2", + "cgroup_peak_bytes": 225161216, + "cgroup_current_bytes": 182919168, + "pid1_vmhwm_bytes": 192196608, + "pid1_vmrss_bytes": 192196608, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0, + "sock_throttled": 0 + } + } + } +} diff --git a/evidence/dependency-failures.json b/evidence/dependency-failures.json new file mode 100644 index 00000000..406167ed --- /dev/null +++ b/evidence/dependency-failures.json @@ -0,0 +1,20 @@ +{ + "result": "PASS", + "redis": { + "profile_status": 200, + "fallback_seconds": 0.0132, + "anonymous": 401, + "readiness": 200 + }, + "postgres": { + "authenticated_status": 503, + "response_seconds": 0.0047, + "anonymous_status": 401, + "internal_api_liveness": [ + 200, + 200 + ], + "note": "Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable." + }, + "same_session_after_recovery": true +} diff --git a/evidence/fresh-migrations.txt b/evidence/fresh-migrations.txt new file mode 100644 index 00000000..43f54020 --- /dev/null +++ b/evidence/fresh-migrations.txt @@ -0,0 +1,20 @@ +./scripts/check_fresh_migrations.sh +Applied 001_initial.sql +Applied 002_idempotency.sql +Applied 003_attachments.sql +Migrations up to date +Migrations up to date +..................... [100%] +=============================== warnings summary =============================== +student-tests/test_attachments.py: 13 warnings +student-tests/test_idempotency.py: 1 warning + /usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() + +student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename + /usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack. + warnings.warn( + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +21 passed, 15 warnings in 4.23s +PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes. diff --git a/evidence/graceful-shutdown.json b/evidence/graceful-shutdown.json new file mode 100644 index 00000000..a324600e --- /dev/null +++ b/evidence/graceful-shutdown.json @@ -0,0 +1,16 @@ +{ + "result": "PASS", + "target_container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c", + "target_instance": "69d3312a6f6b4f7bb50651c8632e196a", + "fixture": "temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards", + "in_flight_observed_in_pg_stat_activity": true, + "in_flight_response": 201, + "new_tcp_after_sigterm": { + "accepted": false, + "error_type": "ConnectionRefusedError" + }, + "target_still_running_when_new_tcp_rejected": true, + "exit_code": 0, + "shutdown_seconds_from_sigterm": 2.0927, + "confirmed_message_preserved": true +} diff --git a/evidence/infrastructure.json b/evidence/infrastructure.json new file mode 100644 index 00000000..6d1642ca --- /dev/null +++ b/evidence/infrastructure.json @@ -0,0 +1,179 @@ +{ + "result": "PASS", + "containers": [ + { + "service": "api", + "container": "102a32862f73ac850881b369827c3d0fef35d41445893bd6807200759f201e3b", + "name": "mtusi-lab4-api-3", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:26:04.73288638Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:e53ef1cc7aef252b12b8f7e0136c8836a5ab075152b194a609e6c6fa1f4de351", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data", + "mtusi-lab4_front" + ], + "http_status": 200, + "instance": "3715a1c3cd074e9f952ad3b9bb646d54", + "release": "lab4-ce567129" + }, + { + "service": "api", + "container": "2089d486fe8dbdf85cfda30cfe2ea79de996f070343cb076dc3bceb83d18e8a0", + "name": "mtusi-lab4-api-4", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:26:10.354473716Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:e53ef1cc7aef252b12b8f7e0136c8836a5ab075152b194a609e6c6fa1f4de351", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data", + "mtusi-lab4_front" + ], + "http_status": 200, + "instance": "0f0ee5b42e754b7f9aa359e746294e8f", + "release": "lab4-ce567129" + }, + { + "service": "cleanup", + "container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6", + "name": "mtusi-lab4-cleanup-1", + "status": "running", + "health": null, + "started_at": "2026-11-02T09:24:53.220476166Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 201326592, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "lb", + "container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71", + "name": "mtusi-lab4-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:24:50.272136138Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab4_front" + ] + }, + { + "service": "migrate", + "container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2", + "name": "mtusi-lab4-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-02T09:25:13.130777342Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "minio", + "container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36", + "name": "mtusi-lab4-minio-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:24:50.261138388Z", + "image": "mtusi-lab4-minio:2025-04-22", + "image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_minio_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "minio-init", + "container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641", + "name": "mtusi-lab4-minio-init-1", + "status": "exited", + "health": null, + "started_at": "2026-11-02T09:25:13.207299134Z", + "image": "mtusi-lab4-mc:2025-04-16", + "image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "postgres", + "container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2", + "name": "mtusi-lab4-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:26:21.713520555Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_postgres_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "redis", + "container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a", + "name": "mtusi-lab4-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:26:20.609059638Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_redis_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + } + ] +} diff --git a/evidence/interrupted-upload.json b/evidence/interrupted-upload.json new file mode 100644 index 00000000..d032ad13 --- /dev/null +++ b/evidence/interrupted-upload.json @@ -0,0 +1,7 @@ +{ + "result": "PASS", + "declared_bytes": 500000, + "sent_bytes": 108, + "metadata_unchanged": true, + "s3_object_count_unchanged": true +} diff --git a/evidence/last-stop.json b/evidence/last-stop.json new file mode 100644 index 00000000..7e13a672 --- /dev/null +++ b/evidence/last-stop.json @@ -0,0 +1,11 @@ +{ + "container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72", + "name": "mtusi-lab4-api-2", + "instance": "6e358c3478904b0e97230b8ceeb17c24", + "release": "lab4-v1", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "stopped_at_utc": "2026-11-02T09:25:37.448855+00:00", + "stop_seconds": 0.5935, + "exit_code": 0, + "left_stopped": true +} diff --git a/evidence/nginx.json b/evidence/nginx.json new file mode 100644 index 00000000..9c2a1fba --- /dev/null +++ b/evidence/nginx.json @@ -0,0 +1,11 @@ +{ + "result": "PASS", + "version": "nginx version: nginx/1.28.0", + "image": "nginx:1.28.0-alpine", + "public_server_header": "nginx", + "config_syntax": "PASS", + "observed_instances": [ + "3ca208a978e441ccba13625143527d66", + "de89643f69534e43bc1111510e0a993e" + ] +} diff --git a/evidence/openapi.txt b/evidence/openapi.txt new file mode 100644 index 00000000..49c347b6 --- /dev/null +++ b/evidence/openapi.txt @@ -0,0 +1,2 @@ +python3 scripts/cluster.py exec python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/orphan-cleanup.json b/evidence/orphan-cleanup.json new file mode 100644 index 00000000..58732e42 --- /dev/null +++ b/evidence/orphan-cleanup.json @@ -0,0 +1,52 @@ +{ + "result": "PASS", + "fault": "SIGKILL API after successful S3PUT, before metadata COMMIT", + "crash_exit_code": 137, + "target_container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c", + "upload_acknowledged": false, + "orphan_confirmed": true, + "grace_wait_seconds": 61, + "dry_run": { + "mode": "dry-run", + "grace_seconds": 60, + "listed": 29, + "fresh": 1, + "tracked": 26, + "locked": 1, + "orphans": 1, + "deleted": 0 + }, + "apply": { + "mode": "apply", + "grace_seconds": 60, + "listed": 29, + "fresh": 1, + "tracked": 26, + "locked": 1, + "orphans": 1, + "deleted": 1 + }, + "repeat": { + "mode": "apply", + "grace_seconds": 60, + "listed": 28, + "fresh": 1, + "tracked": 26, + "locked": 1, + "orphans": 0, + "deleted": 0 + }, + "after_lock_release": { + "mode": "apply", + "grace_seconds": 60, + "listed": 28, + "fresh": 1, + "tracked": 26, + "locked": 0, + "orphans": 1, + "deleted": 1 + }, + "fresh_preserved": true, + "tracked_and_linked_preserved": true, + "active_lock_preserved": true +} diff --git a/evidence/persistence.json b/evidence/persistence.json new file mode 100644 index 00000000..9204f01c --- /dev/null +++ b/evidence/persistence.json @@ -0,0 +1,356 @@ +{ + "result": "PASS", + "before": [ + { + "service": "api", + "container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c", + "name": "mtusi-lab4-api-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:23:34.501426213Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data", + "mtusi-lab4_front" + ], + "http_status": 200, + "instance": "3ca208a978e441ccba13625143527d66", + "release": "lab4-v1" + }, + { + "service": "api", + "container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72", + "name": "mtusi-lab4-api-2", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:15:38.308471337Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data", + "mtusi-lab4_front" + ], + "http_status": 200, + "instance": "de89643f69534e43bc1111510e0a993e", + "release": "lab4-v1" + }, + { + "service": "cleanup", + "container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6", + "name": "mtusi-lab4-cleanup-1", + "status": "running", + "health": null, + "started_at": "2026-11-02T09:10:38.357370005Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 201326592, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "lb", + "container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71", + "name": "mtusi-lab4-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:03:39.330794254Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab4_front" + ] + }, + { + "service": "migrate", + "container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2", + "name": "mtusi-lab4-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-02T09:10:37.212308463Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "minio", + "container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36", + "name": "mtusi-lab4-minio-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:12:16.586994384Z", + "image": "mtusi-lab4-minio:2025-04-22", + "image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_minio_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "minio-init", + "container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641", + "name": "mtusi-lab4-minio-init-1", + "status": "exited", + "health": null, + "started_at": "2026-11-02T09:10:37.288107921Z", + "image": "mtusi-lab4-mc:2025-04-16", + "image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "postgres", + "container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2", + "name": "mtusi-lab4-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:03:32.508777334Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_postgres_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "redis", + "container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a", + "name": "mtusi-lab4-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:03:32.445843167Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_redis_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + } + ], + "after": [ + { + "service": "api", + "container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c", + "name": "mtusi-lab4-api-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:24:50.617456221Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data", + "mtusi-lab4_front" + ], + "http_status": 200, + "instance": "c5dd801e4df142e599da4bafc592e9d7", + "release": "lab4-v1" + }, + { + "service": "api", + "container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72", + "name": "mtusi-lab4-api-2", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:24:50.685437846Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data", + "mtusi-lab4_front" + ], + "http_status": 200, + "instance": "6e358c3478904b0e97230b8ceeb17c24", + "release": "lab4-v1" + }, + { + "service": "cleanup", + "container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6", + "name": "mtusi-lab4-cleanup-1", + "status": "running", + "health": null, + "started_at": "2026-11-02T09:24:53.220476166Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 201326592, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "lb", + "container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71", + "name": "mtusi-lab4-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:24:50.272136138Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab4_front" + ] + }, + { + "service": "migrate", + "container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2", + "name": "mtusi-lab4-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-02T09:24:53.965337083Z", + "image": "mtusi-lab4-api:lab4", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "minio", + "container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36", + "name": "mtusi-lab4-minio-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:24:50.261138388Z", + "image": "mtusi-lab4-minio:2025-04-22", + "image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_minio_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "minio-init", + "container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641", + "name": "mtusi-lab4-minio-init-1", + "status": "exited", + "health": null, + "started_at": "2026-11-02T09:24:53.907159458Z", + "image": "mtusi-lab4-mc:2025-04-16", + "image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "postgres", + "container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2", + "name": "mtusi-lab4-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:24:50.284073804Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_postgres_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + }, + { + "service": "redis", + "container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a", + "name": "mtusi-lab4-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:24:50.227454096Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab4_redis_data" + ], + "networks": [ + "mtusi-lab4_data" + ] + } + ], + "public_test": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence." +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/public-failover.json b/evidence/public-failover.json new file mode 100644 index 00000000..44c156a7 --- /dev/null +++ b/evidence/public-failover.json @@ -0,0 +1,15 @@ +{ + "result": "PASS", + "public_test": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 134, \"failed_attempts\": 0, \"first_recovery_seconds\": 1.104}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "stopped": { + "container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c", + "name": "mtusi-lab4-api-1", + "instance": "c5dd801e4df142e599da4bafc592e9d7", + "release": "lab4-v1", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "stopped_at_utc": "2026-11-02T09:24:58.384960+00:00", + "stop_seconds": 0.4473, + "exit_code": 0, + "left_stopped": true + } +} diff --git a/evidence/public-failover.txt b/evidence/public-failover.txt new file mode 100644 index 00000000..eed9a7cc --- /dev/null +++ b/evidence/public-failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 134, "failed_attempts": 0, "first_recovery_seconds": 1.104} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/rolling-update.json b/evidence/rolling-update.json new file mode 100644 index 00000000..3b03d60e --- /dev/null +++ b/evidence/rolling-update.json @@ -0,0 +1,171 @@ +{ + "result": "PASS", + "before": [ + { + "container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c", + "name": "mtusi-lab4-api-1", + "instance": "0e4e9b12996647ac8d03d03d79ae64fe", + "release": "lab4-v1", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961" + }, + { + "container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72", + "name": "mtusi-lab4-api-2", + "instance": "427b43f714ff4bc3b99698dc90cafa3b", + "release": "lab4-v1", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961" + } + ], + "after": [ + { + "container": "102a32862f73ac850881b369827c3d0fef35d41445893bd6807200759f201e3b", + "name": "mtusi-lab4-api-3", + "instance": "3715a1c3cd074e9f952ad3b9bb646d54", + "release": "lab4-ce567129", + "image_id": "sha256:e53ef1cc7aef252b12b8f7e0136c8836a5ab075152b194a609e6c6fa1f4de351" + }, + { + "container": "2089d486fe8dbdf85cfda30cfe2ea79de996f070343cb076dc3bceb83d18e8a0", + "name": "mtusi-lab4-api-4", + "instance": "0f0ee5b42e754b7f9aa359e746294e8f", + "release": "lab4-ce567129", + "image_id": "sha256:e53ef1cc7aef252b12b8f7e0136c8836a5ab075152b194a609e6c6fa1f4de351" + } + ], + "target_release": "lab4-ce567129", + "lb_before": { + "service": "lb", + "container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71", + "name": "mtusi-lab4-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:24:50.272136138Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab4_front" + ] + }, + "lb_after": { + "service": "lb", + "container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71", + "name": "mtusi-lab4-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T09:24:50.272136138Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab4_front" + ] + }, + "lb_not_restarted": true, + "build_seconds": 2.7115, + "migration_output": "Migrations up to date", + "replacements": [ + { + "old_container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c", + "new_container": "102a32862f73ac850881b369827c3d0fef35d41445893bd6807200759f201e3b", + "old_ip": "172.25.0.2", + "new_ip": "172.25.0.5", + "dns_discovered_new_address": true, + "instances_during_start_first": [ + "0e4e9b12996647ac8d03d03d79ae64fe", + "3715a1c3cd074e9f952ad3b9bb646d54", + "427b43f714ff4bc3b99698dc90cafa3b" + ], + "observed_instances": [ + "3715a1c3cd074e9f952ad3b9bb646d54", + "427b43f714ff4bc3b99698dc90cafa3b" + ], + "old_exit_code": 0, + "seconds": 5.6289 + }, + { + "old_container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72", + "new_container": "2089d486fe8dbdf85cfda30cfe2ea79de996f070343cb076dc3bceb83d18e8a0", + "old_ip": "172.25.0.3", + "new_ip": "172.25.0.2", + "dns_discovered_new_address": true, + "instances_during_start_first": [ + "0f0ee5b42e754b7f9aa359e746294e8f", + "3715a1c3cd074e9f952ad3b9bb646d54", + "427b43f714ff4bc3b99698dc90cafa3b" + ], + "observed_instances": [ + "0f0ee5b42e754b7f9aa359e746294e8f", + "3715a1c3cd074e9f952ad3b9bb646d54" + ], + "old_exit_code": 0, + "seconds": 5.4987 + } + ], + "during_replacement": { + "seconds": 11.1277, + "concurrency": 4, + "successful_requests": 8499, + "failed_attempts": 0, + "successful_rps": 763.77, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 4.34, + "p95": 7.505, + "p99": 37.719 + }, + "instances": [ + "0e4e9b12996647ac8d03d03d79ae64fe", + "0f0ee5b42e754b7f9aa359e746294e8f", + "3715a1c3cd074e9f952ad3b9bb646d54", + "427b43f714ff4bc3b99698dc90cafa3b" + ] + }, + "total": { + "seconds": 20.7958, + "concurrency": 4, + "successful_requests": 15875, + "failed_attempts": 0, + "successful_rps": 763.377, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 4.479, + "p95": 7.406, + "p99": 28.635 + }, + "instances": [ + "0e4e9b12996647ac8d03d03d79ae64fe", + "0f0ee5b42e754b7f9aa359e746294e8f", + "3715a1c3cd074e9f952ad3b9bb646d54", + "427b43f714ff4bc3b99698dc90cafa3b" + ] + }, + "data": { + "confirmed_unique_messages": 15875, + "history_messages": 15875, + "confirmed_preserved": true, + "duplicate_ids": 0, + "duplicate_texts": 0 + }, + "change": "New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate." +} diff --git a/evidence/s3-outage.json b/evidence/s3-outage.json new file mode 100644 index 00000000..1def0adf --- /dev/null +++ b/evidence/s3-outage.json @@ -0,0 +1,7 @@ +{ + "result": "PASS", + "upload_status_during_outage": 503, + "response_seconds": 0.2487, + "original_preserved_after_restart": true, + "same_session_after_recovery": true +} diff --git a/evidence/s3-security.json b/evidence/s3-security.json new file mode 100644 index 00000000..82f8b3f4 --- /dev/null +++ b/evidence/s3-security.json @@ -0,0 +1,10 @@ +{ + "result": "PASS", + "anonymous_get": 403, + "create_bucket_with_api_credentials": 403, + "list_other_prefix": 403, + "console_reachable_from_api": false, + "root_secret_mounted_in_api": false, + "bootstrap_repeated_without_data_loss": true, + "bootstrap_output": "PASS: private bucket and restricted API policy ready (credentials hidden)." +} diff --git a/evidence/sequential-failover.json b/evidence/sequential-failover.json new file mode 100644 index 00000000..a66ee4ab --- /dev/null +++ b/evidence/sequential-failover.json @@ -0,0 +1,135 @@ +{ + "result": "PASS", + "baseline": { + "seconds": 3.0101, + "concurrency": 4, + "successful_requests": 2029, + "failed_attempts": 0, + "successful_rps": 674.073, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 4.838, + "p95": 7.446, + "p99": 24.831 + }, + "instances": [ + "118d370a438c4c14aabe117988bc86e9", + "6e358c3478904b0e97230b8ceeb17c24" + ] + }, + "phases": [ + { + "stopped": { + "container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c", + "name": "mtusi-lab4-api-1", + "instance": "118d370a438c4c14aabe117988bc86e9", + "release": "lab4-v1", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "stopped_at_utc": "2026-11-02T09:25:23.676868+00:00", + "stop_seconds": 0.5348, + "exit_code": 0, + "left_stopped": true + }, + "traffic": { + "seconds": 8.0006, + "concurrency": 4, + "successful_requests": 3857, + "failed_attempts": 0, + "successful_rps": 482.09, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 6.513, + "p95": 16.08, + "p99": 56.428 + }, + "instances": [ + "118d370a438c4c14aabe117988bc86e9", + "6e358c3478904b0e97230b8ceeb17c24" + ] + }, + "post_stop_write_seconds_from_action_start": 1.2915, + "post_stop_write_seconds_from_stop_completion": 0.0308, + "read_during_outage": { + "status": 200, + "confirmed_message_preserved": true, + "seconds_from_action_start": 1.2842, + "seconds_from_stop_completion": 0.0235 + }, + "new_connection_after_stop": { + "accepted": false, + "error_type": "TimeoutError" + }, + "stopped_remains_stopped": true + }, + { + "stopped": { + "container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72", + "name": "mtusi-lab4-api-2", + "instance": "6e358c3478904b0e97230b8ceeb17c24", + "release": "lab4-v1", + "image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961", + "stopped_at_utc": "2026-11-02T09:25:37.448855+00:00", + "stop_seconds": 0.5935, + "exit_code": 0, + "left_stopped": true + }, + "traffic": { + "seconds": 8.0018, + "concurrency": 4, + "successful_requests": 4137, + "failed_attempts": 1, + "successful_rps": 517.011, + "error_ratio": 0.000242, + "successful_latency_ms": { + "p50": 6.55, + "p95": 14.11, + "p99": 53.684 + }, + "instances": [ + "69d3312a6f6b4f7bb50651c8632e196a", + "6e358c3478904b0e97230b8ceeb17c24" + ] + }, + "post_stop_write_seconds_from_action_start": 1.2502, + "post_stop_write_seconds_from_stop_completion": 0.0189, + "read_during_outage": { + "status": 200, + "confirmed_message_preserved": true, + "seconds_from_action_start": 1.2386, + "seconds_from_stop_completion": 0.0073 + }, + "new_connection_after_stop": { + "accepted": false, + "error_type": "TimeoutError" + }, + "stopped_remains_stopped": true + } + ], + "total": { + "seconds": 30.9392, + "concurrency": 4, + "successful_requests": 17688, + "failed_attempts": 1, + "successful_rps": 571.702, + "error_ratio": 5.7e-05, + "successful_latency_ms": { + "p50": 5.77, + "p95": 12.069, + "p99": 46.873 + }, + "instances": [ + "118d370a438c4c14aabe117988bc86e9", + "427b43f714ff4bc3b99698dc90cafa3b", + "69d3312a6f6b4f7bb50651c8632e196a", + "6e358c3478904b0e97230b8ceeb17c24" + ] + }, + "data": { + "confirmed_unique_messages": 17688, + "history_messages": 17688, + "confirmed_preserved": true, + "duplicate_ids": 0, + "duplicate_texts": 0 + }, + "each_original_container_tested": true +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..1e7b9ff4 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..9fae02d4 --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,14 @@ +python3 scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache +..................... [100%] +=============================== warnings summary =============================== +student-tests/test_attachments.py: 13 warnings +student-tests/test_idempotency.py: 1 warning + /usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() + +student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename + /usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack. + warnings.warn( + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +21 passed, 15 warnings in 4.15s diff --git a/infra/minio/Dockerfile b/infra/minio/Dockerfile new file mode 100644 index 00000000..d1a5fca4 --- /dev/null +++ b/infra/minio/Dockerfile @@ -0,0 +1,28 @@ +# Official public images/binaries were removed upstream; build fixed release sources. +FROM golang:1.24.2-bookworm AS minio-build +ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local +RUN git clone --depth 1 --branch RELEASE.2025-04-22T22-12-26Z https://github.com/minio/minio.git /src +WORKDIR /src +RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/minio . +RUN test "$(git rev-parse HEAD)" = 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e' -o /out/minio . + +FROM golang:1.24.2-bookworm AS mc-build +ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local +RUN git clone --depth 1 --branch RELEASE.2025-04-16T18-13-26Z https://github.com/minio/mc.git /src +WORKDIR /src +RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/mc . +RUN test "$(git rev-parse HEAD)" = b00526b153a31b36767991a4f5ce2cced435ee8e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/mc/cmd.Version=2025-04-16T18:13:26Z -X github.com/minio/mc/cmd.ReleaseTag=RELEASE.2025-04-16T18-13-26Z -X github.com/minio/mc/cmd.CommitID=b00526b153a31b36767991a4f5ce2cced435ee8e' -o /out/mc . + +FROM python:3.13.5-slim AS minio +LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.version="RELEASE.2025-04-22T22-12-26Z" +RUN useradd --uid 10002 --create-home minio && mkdir /data && chown 10002:10002 /data +COPY --from=minio-build /out/minio /usr/local/bin/minio +COPY infra/minio/server.sh /server.sh +USER 10002 +ENTRYPOINT ["/bin/sh", "/server.sh"] +CMD ["server", "/data", "--address", ":9000", "--console-address", "127.0.0.1:9001"] + +FROM debian:bookworm-slim AS mc +LABEL org.opencontainers.image.source="https://github.com/minio/mc" org.opencontainers.image.version="RELEASE.2025-04-16T18-13-26Z" +COPY --from=mc-build /out/mc /usr/local/bin/mc +ENTRYPOINT ["/bin/sh", "/config/bootstrap.sh"] diff --git a/infra/minio/api-policy.json b/infra/minio/api-policy.json new file mode 100644 index 00000000..01d0005b --- /dev/null +++ b/infra/minio/api-policy.json @@ -0,0 +1,8 @@ +{ + "Version": "2012-10-17", + "Statement": [ + {"Effect":"Allow","Action":["s3:GetBucketLocation"],"Resource":["arn:aws:s3:::messenger-attachments"]}, + {"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::messenger-attachments"],"Condition":{"StringLike":{"s3:prefix":["originals/*"]}}}, + {"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::messenger-attachments/originals/*"]} + ] +} diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh new file mode 100644 index 00000000..7d9eb39d --- /dev/null +++ b/infra/minio/bootstrap.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +mc alias set local http://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null +mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null +mc admin policy create local messenger-api /config/api-policy.json >/dev/null +mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null +printf '%s\n' 'PASS: private bucket and restricted API policy ready (credentials hidden).' diff --git a/infra/minio/server.sh b/infra/minio/server.sh new file mode 100644 index 00000000..aedb392b --- /dev/null +++ b/infra/minio/server.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +export MINIO_ROOT_USER="$(cat /run/secrets/minio_root_user)" +export MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)" +exec minio "$@" diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf new file mode 100644 index 00000000..5f0b43d3 --- /dev/null +++ b/infra/nginx/nginx.conf @@ -0,0 +1,62 @@ +worker_processes 1; +error_log /dev/stderr warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + access_log off; + server_tokens off; + limit_conn_zone $server_name zone=uploads:64k; + + # Docker DNS; resolve updates the upstream when replicas get new IPs. + resolver 127.0.0.11 valid=1s ipv6=off; + resolver_timeout 1s; + + upstream messenger { + zone messenger 64k; + # All A records of api participate in the default round-robin balancing. + server api:8080 resolve max_fails=1 fail_timeout=2s; + } + + server { + listen 8080; + server_name messenger; + + location ~ ^/api/v1/chats/[^/]+/attachments$ { + limit_conn uploads 16; + client_max_body_size 11m; + client_body_buffer_size 64k; + client_body_timeout 10s; + proxy_request_buffering on; + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header Connection ""; + proxy_connect_timeout 1s; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + proxy_next_upstream error timeout http_503; + proxy_next_upstream_tries 2; + } + + location / { + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header Connection ""; + + proxy_connect_timeout 1s; + proxy_read_timeout 5s; + proxy_send_timeout 5s; + + # Passive checks. Do NOT enable non_idempotent: an already-sent POST + # must not be resent by Nginx after an ambiguous response. + proxy_next_upstream error timeout http_503; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 6s; + } + } +} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..7b8c5338 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,34 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id); +CREATE INDEX chats_history_idx ON chats(created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id); +CREATE TABLE sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry_idx ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..855e66a8 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,11 @@ +-- Additive: old lab2 tables and messages remain valid. +CREATE TABLE idempotency_records ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash text NOT NULL, + message_id uuid REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, key) +); +CREATE INDEX idempotency_expiry_idx ON idempotency_records(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..a3de2f85 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,23 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id) +); +CREATE INDEX attachments_chat_idx ON attachments(chat_id,id); +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000); +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id), + attachment_id uuid NOT NULL REFERENCES attachments(id), + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id,attachment_id), + UNIQUE (message_id,position) +); diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 00000000..78da0202 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,51 @@ +annotated-types==0.8.0 +anyio==4.9.0 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +attrs==26.1.0 +boto3==1.38.27 +botocore==1.38.46 +certifi==2026.10.23 +cffi==2.1.1 +charset-normalizer==3.5.2 +click==8.5.0 +fastapi==0.115.12 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.20 +iniconfig==2.3.1 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-path==0.3.4 +jsonschema-specifications==2025.9.1 +lazy-object-proxy==1.13.0 +openapi-schema-validator==0.6.3 +openapi-spec-validator==0.7.2 +packaging==26.3 +pathable==0.4.4 +pillow==11.3.0 +pluggy==1.7.0 +psycopg==3.2.9 +psycopg-binary==3.2.9 +psycopg-pool==3.2.6 +pycparser==3.12 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.21.0 +pytest==8.4.0 +python-dateutil==2.9.0.post0 +PyYAML==6.0.3 +redis==6.2.0 +referencing==0.36.2 +requests==2.34.2 +rfc3339-validator==0.1.4 +rpds-py==2026.9.1 +s3transfer==0.13.1 +six==1.17.0 +sniffio==1.3.1 +starlette==0.46.2 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.34.3 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..d7c23069 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +fastapi==0.115.12 +uvicorn==0.34.3 +psycopg[binary]==3.2.9 +psycopg-pool==3.2.6 +redis==6.2.0 +argon2-cffi==25.1.0 +pytest==8.4.0 +httpx==0.28.1 +pydantic==2.11.5 +anyio==4.9.0 +boto3==1.38.27 +Pillow==11.3.0 diff --git a/scripts/attachment_checks.py b/scripts/attachment_checks.py new file mode 100644 index 00000000..6731a3ff --- /dev/null +++ b/scripts/attachment_checks.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Container-side privileged test utilities. Input over stdin; never output credentials.""" +import base64 +from datetime import datetime,timezone +import io +import json +from pathlib import Path +import socket +import sys +import uuid + +import boto3 +from botocore import UNSIGNED +from botocore.config import Config +from botocore.exceptions import ClientError +import psycopg +from psycopg import sql +from psycopg.rows import dict_row +from messenger.db import setting,Database +from messenger.object_store import ObjectStore +from messenger.cleanup import sweep + + +def main(): + p=json.load(sys.stdin) + action=p['action'] + store=ObjectStore() + result={} + with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row) as conn: + if action=='lookup': + result=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(p['id'],)).fetchone() + elif action=='counts': + result={'attachments':conn.execute('SELECT count(*) AS n FROM attachments').fetchone()['n'], + 'objects':sum(len(page.get('Contents',[])) for page in store.pages())} + elif action=='list': + result={'keys':[obj['Key'] for page in store.pages() for obj in page.get('Contents',[])]} + elif action=='put': + raw=base64.b64decode(p['bytes']) + store.put(p['key'],io.BytesIO(raw),len(raw),'image/png') + result={'created':True} + elif action=='delete': + store.delete(p['key']) + result={'deleted':True} + elif action=='exists': + try: + obj=store.client.head_object(Bucket=store.bucket,Key=p['key']) + result={'exists':True,'length':obj['ContentLength']} + except ClientError as exc: + if exc.response['ResponseMetadata']['HTTPStatusCode']!=404: + raise + result={'exists':False} + elif action=='tracked': + result={'tracked':conn.execute('SELECT count(*) AS n FROM attachments WHERE object_key=%s',(p['key'],)).fetchone()['n']} + elif action=='security': + unsigned=boto3.client('s3',endpoint_url=setting('S3_ENDPOINT'),region_name='us-east-1',config=Config(signature_version=UNSIGNED,s3={'addressing_style':'path'},connect_timeout=1,read_timeout=3,retries={'total_max_attempts':1})) + try: + unsigned.get_object(Bucket=store.bucket,Key=p['key']) + anonymous=200 + except ClientError as exc: + anonymous=exc.response['ResponseMetadata']['HTTPStatusCode'] + try: + store.client.create_bucket(Bucket='forbidden-'+uuid.uuid4().hex) + create_bucket=200 + except ClientError as exc: + create_bucket=exc.response['ResponseMetadata']['HTTPStatusCode'] + try: + store.client.list_objects_v2(Bucket=store.bucket,Prefix='forbidden/') + other_prefix=200 + except ClientError as exc: + other_prefix=exc.response['ResponseMetadata']['HTTPStatusCode'] + try: + sock=socket.create_connection(('minio',9001),timeout=1) + sock.close() + console=True + except OSError: + console=False + result={'anonymous_get':anonymous,'create_bucket_with_api_credentials':create_bucket, + 'list_other_prefix':other_prefix,'console_reachable_from_api':console, + 'root_secret_mounted_in_api':Path('/run/secrets/minio_root_password').exists()} + unsigned.close() + elif action=='install': + conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename={} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['filename']))) + conn.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function']))) + result={'installed':True} + elif action=='remove': + conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON attachments').format(sql.Identifier(p['trigger']))) + conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function']))) + result={'removed':True} + elif action=='waiting': + result={'waiting':conn.execute("SELECT count(*) AS n FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO attachments%%'").fetchone()['n']} + elif action=='sweep': + db=Database() + db.open() + try: + result=sweep(db,store,grace_seconds=p.get('grace',60),apply=p.get('apply',False)) + finally: + db.close() + else: + raise ValueError('Unknown check action') + store.close() + print(json.dumps(result,default=str)) + + +if __name__=='__main__': + main() diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 00000000..010bc7ce --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,32 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets +python3 - <<'PY' +from pathlib import Path +import secrets +root = Path('secrets') +p = root / 'postgres_password' +if not p.exists(): + p.write_text(secrets.token_hex(24)) +pw = p.read_text().strip() +for name, value in { + 'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger', + 'cache_url': 'redis://redis:6379/0', + 'minio_root_user': 'lab4-admin', + 'minio_root_password': secrets.token_hex(24), + 's3_access_key_id': 'messenger-api', + 's3_secret_access_key': secrets.token_hex(24), +}.items(): + path = root / name + if not path.exists(): + path.write_text(value) +# Bind-mounted Docker secrets must be readable by container uid 10001. +# Parent directory remains 0700 on the host. +for path in root.iterdir(): + path.chmod(0o444) +if not Path('.env').exists(): + Path('.env').write_text('SESSION_TTL_SECONDS=3600\n') +print('Local secrets ready (values hidden).') +PY diff --git a/scripts/capture_load.py b/scripts/capture_load.py new file mode 100644 index 00000000..8b411dff --- /dev/null +++ b/scripts/capture_load.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Capture repeatable short load run and sanitized environment/resource evidence.""" +import datetime +import json +import platform +import re +import subprocess +import sys +import time +import cluster +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def output(args): + return subprocess.check_output(args, cwd=ROOT, text=True).strip() + + +def counts(): + code = "import json,psycopg; from messenger.db import setting; c=psycopg.connect(setting('DATABASE_URL')); print(json.dumps({t:c.execute('SELECT count(*) FROM '+t).fetchone()[0] for t in ['users','chats','messages','sessions','idempotency_records']})); c.close()" + return json.loads(output(['docker','exec',cluster.container_ids('api')[0],'python','-c',code])) + + +def resource_summary(samples): + peaks = {} + for sample in samples: + for container in sample['containers']: + peak = peaks.setdefault(container['Name'], {'cpu_percent_peak':0, 'memory_mib_peak':0}) + peak['cpu_percent_peak'] = max(peak['cpu_percent_peak'],float(container['CPUPerc'].rstrip('%'))) + value,unit = re.match(r'([0-9.]+)([A-Za-z]+)',container['MemUsage']).groups() + multiplier = {'GiB':1024,'MiB':1,'KiB':1/1024,'B':1/1048576}[unit] + peak['memory_mib_peak'] = round(max(peak['memory_mib_peak'],float(value)*multiplier),3) + return {'samples':len(samples),'per_container_peaks':peaks} + + +def main(): + names = cluster.container_ids() + environment = { + 'captured_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'host_os': platform.platform(), 'host_architecture': platform.machine(), + 'docker': json.loads(output(['docker','info','--format','{"server_version":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"architecture":"{{.Architecture}}"}'])), + 'python_in_image': output(['docker','exec',cluster.container_ids('api')[0],'python','--version']), + 'dataset_before': counts(), + 'limits': {'api_each':'1 CPU / 512 MiB','postgres':'1 CPU / 512 MiB','redis':'0.25 CPU / 128 MiB','lb':'0.5 CPU / 128 MiB'}, + 'warmup': 'API already healthy; acceptance, persistence and dependency checks performed before benchmark. No separate timed load warmup.', + 'replicas':len(cluster.replicas()), 'repetitions': 1, 'duration_seconds':30, 'concurrency':4, + } + if sys.platform == 'darwin': + environment['host_cpu'] = output(['sysctl','-n','machdep.cpu.brand_string']) + environment['host_memory_bytes'] = int(output(['sysctl','-n','hw.memsize'])) + samples = [] + with (ROOT/'evidence/load-output.txt').open('w') as log: + proc = subprocess.Popen([sys.executable,'tests/load.py','--duration','30','--concurrency','4','--output','evidence/load.json'],cwd=ROOT,stdout=log,stderr=subprocess.STDOUT) + while proc.poll() is None: + rows = output(['docker','stats','--no-stream','--format','{{json .}}',*names]) + samples.append({'at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'containers':[{k:v for k,v in json.loads(row).items() if k in ('Name','CPUPerc','MemUsage','MemPerc','PIDs')} + for row in rows.splitlines()]}) + time.sleep(1) + if proc.returncode: + raise SystemExit(proc.returncode) + environment['dataset_after'] = counts() + (ROOT/'evidence/environment.json').write_text(json.dumps(environment,ensure_ascii=False,indent=2)+'\n') + (ROOT/'evidence/resources.json').write_text(json.dumps(samples,indent=2)+'\n') + (ROOT/'evidence/resource-summary.json').write_text(json.dumps(resource_summary(samples),indent=2)+'\n') + print((ROOT/'evidence/load.json').read_text()) + + +if __name__ == '__main__': + main() diff --git a/scripts/capture_uploads.py b/scripts/capture_uploads.py new file mode 100644 index 00000000..3cf840fa --- /dev/null +++ b/scripts/capture_uploads.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Ten overlapping PNG uploads with kernel-recorded memory peaks; stdlib only. + +Recreates API containers (never volumes) to reset memory high-water marks. +Run from any directory: python3 scripts/capture_uploads.py. +""" +import concurrent.futures +import datetime +import hashlib +import json +import platform +import struct +import subprocess +import sys +import threading +import time +import urllib.parse +import zlib +from pathlib import Path + +import cluster + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check # noqa: E402 + +WORKERS = 10 +WIDTH, HEIGHT = 3000, 1000 +EVIDENCE = ROOT / 'evidence/concurrent-uploads.json' + + +def make_png(): + """Stable, incompressible RGB rows without a host image library.""" + def chunk(kind, data): + return (struct.pack('!I', len(data)) + kind + data + + struct.pack('!I', zlib.crc32(kind + data) & 0xffffffff)) + + compressor = zlib.compressobj(level=0) + compressed = [] + for row in range(HEIGHT): + pixels = hashlib.shake_256(b'lab4-concurrent-upload-v1:' + str(row).encode()).digest(WIDTH * 3) + compressed.append(compressor.compress(b'\x00' + pixels)) + compressed.append(compressor.flush()) + return (b'\x89PNG\r\n\x1a\n' + + chunk(b'IHDR', struct.pack('!IIBBBBB', WIDTH, HEIGHT, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', b''.join(compressed)) + chunk(b'IEND', b'')) + + +def memory(container): + # cgroup peaks include allocations between observations; VmHWM is PID 1's + # resident peak, whereas cgroup memory also includes file cache and children. + code = '''import json,pathlib +def number(path): + return int(pathlib.Path(path).read_text().strip()) +root=pathlib.Path('/sys/fs/cgroup') +if (root/'memory.peak').exists(): + source='cgroup-v2' + peak=number(root/'memory.peak'); current=number(root/'memory.current') + events=dict((k,int(v)) for k,v in (line.split() for line in (root/'memory.events').read_text().splitlines())) +else: + root=root/'memory'; source='cgroup-v1' + peak=number(root/'memory.max_usage_in_bytes'); current=number(root/'memory.usage_in_bytes') + events={'failcnt':number(root/'memory.failcnt')} +status=dict(line.split(':',1) for line in pathlib.Path('/proc/1/status').read_text().splitlines() if ':' in line) +print(json.dumps({'source':source,'cgroup_peak_bytes':peak,'cgroup_current_bytes':current, + 'pid1_vmhwm_bytes':int(status['VmHWM'].split()[0])*1024, + 'pid1_vmrss_bytes':int(status['VmRSS'].split()[0])*1024,'memory_events':events})) +''' + return json.loads(cluster.command(['docker', 'exec', container, 'python', '-c', code])) + + +def version_context(): + revision = cluster.command(['git', 'rev-parse', 'HEAD']) + dirty = subprocess.run(['git', 'diff', '--quiet'], cwd=ROOT, check=False).returncode != 0 + address = urllib.parse.urlsplit(cluster.BASE_URL) + return { + 'git_commit': revision, 'tracked_worktree_dirty': dirty, + 'host_python': platform.python_version(), 'host_os': platform.platform(), + 'docker_version': cluster.command(['docker', 'version', '--format', '{{.Server.Version}}']), + 'compose_version': cluster.compose('version', '--short'), + 'contract_version': json.loads((ROOT / 'lab.json').read_text())['contract_version'], + 'endpoint': {'scheme': address.scheme, 'host': address.hostname, 'port': address.port}, + } + + +def experiment(evidence): + payload = make_png() + check(9_000_000 <= len(payload) <= 10 * 1024 * 1024, 'generated PNG size outside experiment bounds') + digest = hashlib.sha256(payload).hexdigest() + evidence.update({ + 'context': version_context(), + 'input': {'format': 'PNG', 'mode': 'RGB', 'width': WIDTH, 'height': HEIGHT, + 'size_bytes': len(payload), 'sha256': digest, + 'generator': 'SHAKE-256 lab4-concurrent-upload-v1 row seeds; filter 0; zlib level 0', + 'same_immutable_bytes_for_all_requests': True}, + 'concurrency': WORKERS, + 'reset': {'command': 'docker compose up -d --no-deps --force-recreate api', + 'purpose': 'Reset API cgroup memory.peak and PID 1 VmHWM; retain all volumes'}, + 'peak_scope': 'Fresh API startup, bootstrap, ten uploads and representative download; no peak sampling gaps', + }) + cluster.compose('up', '-d', '--no-deps', '--force-recreate', 'api') + containers = cluster.container_ids('api') + check(len(containers) == 2, 'experiment requires exactly two API containers') + for container in containers: + cluster.wait_healthy(container) + cluster.wait_cluster(2) + replicas = cluster.replicas() + evidence['cluster_before'] = cluster.snapshot() + evidence['baseline'] = {item['instance']: memory(item['container']) for item in replicas} + limits = [item for item in evidence['cluster_before'] if item['service'] == 'api'] + evidence['memory_limits'] = [ + {key: item[key] for key in ('instance', 'memory_bytes', 'nano_cpus')} for item in limits] + check(all(item['memory_bytes'] == 512 * 1024 * 1024 and item['nano_cpus'] == 1_000_000_000 + for item in limits), 'API limits must be 512 MiB and 1 CPU each') + client = Client(timeout=180) + alice, bob, _eve, chat = client.bootstrap() + barrier = threading.Barrier(WORKERS, timeout=30) + lock = threading.Lock() + activity = {'active': 0, 'peak': 0} + origin = time.monotonic() + + def upload(index): + worker = Client(timeout=180) + barrier.wait() + with lock: + start = time.monotonic() - origin + activity['active'] += 1 + activity['peak'] = max(activity['peak'], activity['active']) + item = {'index': index, 'start_seconds': start} + attachment = None + try: + response = worker.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=concurrent-%02d.png' % index, + expected=201, actor=alice, raw=payload, + headers={'Content-Type': 'image/png'}, schema='Attachment') + attachment = response['json'] + item.update({'status': response['status'], 'instance': response['headers'].get('X-Instance-Id')}) + check(attachment['sha256'] == digest and attachment['size_bytes'] == len(payload), + 'uploaded original checksum/size mismatch') + check(attachment['chat_id'] == chat['id'] and attachment['owner_id'] == alice['id'], + 'uploaded attachment ownership mismatch') + check(attachment['status'] == 'ready', 'uploaded attachment not ready') + check(attachment['content_type'] == 'image/png', 'uploaded MIME type mismatch') + item.update({'attachment_id': attachment['id'], 'metadata_verified': True}) + except Failure as exc: + item['error'] = str(exc) + except Exception as exc: + # Never serialize arbitrary library exception text, request or actor. + item['error'] = type(exc).__name__ + finally: + with lock: + item['end_seconds'] = time.monotonic() - origin + activity['active'] -= 1 + return item, attachment + + with concurrent.futures.ThreadPoolExecutor(max_workers=WORKERS) as pool: + completed = list(pool.map(upload, range(WORKERS))) + requests = [item for item, _attachment in completed] + evidence['requests'] = requests + evidence['elapsed_seconds'] = time.monotonic() - origin + evidence['peak_active_requests'] = activity['peak'] + overlap = min(item['end_seconds'] for item in requests) - max(item['start_seconds'] for item in requests) + evidence['all_ten_overlap_seconds'] = max(0, overlap) + evidence['all_ten_overlapping'] = overlap > 0 + evidence['successful_count'] = sum('error' not in item for item in requests) + evidence['error_count'] = WORKERS - evidence['successful_count'] + evidence['observed_instances'] = sorted({item['instance'] for item in requests if item.get('instance')}) + successful = [attachment for item, attachment in completed if 'error' not in item] + if successful: + download = client.call('GET', '/api/v1/attachments/' + successful[0]['id'] + '/content', actor=bob) + check(download['bytes'] == payload, 'representative download differs from original') + evidence['representative_download'] = { + 'verified_as': 'bob (chat member)', 'matches_original_bytes': True, + 'size_bytes': len(download['bytes']), 'sha256': hashlib.sha256(download['bytes']).hexdigest()} + evidence['final'] = {item['instance']: memory(item['container']) for item in replicas} + evidence['cluster_after'] = cluster.snapshot() + states = cluster.inspect(containers) + evidence['oom_killed'] = {item['Name'].lstrip('/'): item['State']['OOMKilled'] for item in states} + evidence['per_node_peaks'] = evidence['final'] + check(evidence['successful_count'] == WORKERS, 'not all ten uploads succeeded') + check(len(evidence['observed_instances']) >= 2, 'uploads did not reach both API instances') + check(overlap > 0 and activity['peak'] == WORKERS, 'ten requests did not all overlap') + check(not any(evidence['oom_killed'].values()), 'an API container was OOM-killed') + for item in replicas: + before, after = evidence['baseline'][item['instance']], evidence['final'][item['instance']] + check(after['cgroup_peak_bytes'] <= 512 * 1024 * 1024, 'API exceeded configured memory limit') + for event in ('oom', 'oom_kill', 'failcnt'): + check(before['memory_events'].get(event, 0) == 0 and after['memory_events'].get(event, 0) == 0, + 'API recorded an OOM or cgroup allocation failure') + check(all(item['State']['Running'] and item['State'].get('Health', {}).get('Status') == 'healthy' + for item in states), 'both APIs must remain running and healthy') + cluster.wait_cluster(2) + + +def main(): + evidence = {'captured_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'result': 'FAIL'} + try: + experiment(evidence) + evidence['result'] = 'PASS' + except Failure as exc: + evidence['failure'] = str(exc) + except Exception as exc: + evidence['failure'] = type(exc).__name__ + finally: + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + EVIDENCE.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n') + print(evidence['result'] + ': evidence/concurrent-uploads.json') + if evidence['result'] != 'PASS': + print(evidence.get('failure', 'experiment failed')) + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/scripts/check_fresh_migrations.sh b/scripts/check_fresh_migrations.sh new file mode 100755 index 00000000..776056a1 --- /dev/null +++ b/scripts/check_fresh_migrations.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# A unique disposable Compose project; never touches normal project volumes. +set -eu +cd "$(dirname "$0")/.." +project="mtusi-lab4-migration-check-$$" +if [ -n "$(docker compose -p "$project" ps -aq)" ]; then + echo 'Refusing to reuse a pre-existing scratch project' >&2 + exit 1 +fi +cleanup() { + docker compose -p "$project" down --volumes >/dev/null 2>&1 +} +trap cleanup EXIT INT TERM +docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis minio minio-init +docker compose -p "$project" run --rm --no-deps api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache' +echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.' diff --git a/scripts/cluster.py b/scripts/cluster.py new file mode 100644 index 00000000..c8ed6724 --- /dev/null +++ b/scripts/cluster.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence.""" +import argparse +import datetime +import json +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/') + + +def command(args, timeout=120): + return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip() + + +def compose(*args, timeout=120): + return command(['docker', 'compose', *args], timeout=timeout) + + +def inspect(containers): + if not containers: + return [] + return json.loads(command(['docker','inspect',*containers])) + + +def container_ids(service=None, all_containers=False): + args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else []) + return compose(*args).split() + + +def probe(container, host='127.0.0.1', path='/health/live'): + code = '''import json,sys,urllib.request,urllib.error +try: + r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4) +except urllib.error.HTTPError as exc: + r=exc +print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')})) +''' + return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8)) + + +def replicas(): + items = [] + for data in inspect(container_ids('api')): + if data['State']['Running']: + result = probe(data['Id']) + items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'), + 'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']}) + return items + + +def snapshot(): + items = [] + for data in inspect(container_ids(all_containers=True)): + service = data['Config']['Labels']['com.docker.compose.service'] + item = { + 'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'), + 'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'), + 'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'], + 'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'], + 'published_ports':data['HostConfig']['PortBindings'], + 'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'), + 'networks':sorted(data['NetworkSettings']['Networks']), + } + if service == 'api' and data['State']['Running']: + result = probe(data['Id']) + item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']}) + items.append(item) + return items + + +def wait_healthy(container, timeout=60): + deadline = time.monotonic()+timeout + while time.monotonic() < deadline: + data = inspect([container])[0] + if data['State'].get('Health',{}).get('Status') == 'healthy': + return + time.sleep(0.3) + raise RuntimeError('Container did not become healthy') + + +def wait_cluster(count=2, timeout=60): + deadline = time.monotonic()+timeout + seen = set() + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response: + alias = response.headers.get('X-Instance-Id') + if alias: + seen.add(alias) + if len(seen) >= count: + return sorted(seen) + except (OSError, urllib.error.URLError): + pass + time.sleep(0.1) + raise RuntimeError('Load balancer did not discover required API instances') + + +def stop_one(target): + if not re.fullmatch(r'[a-f0-9]{32}', target): + raise ValueError('Invalid target instance alias') + matching = [item for item in replicas() if item['instance']==target] + if len(matching) != 1: + raise RuntimeError('Target must match exactly one running API process') + item = matching[0] + started = time.monotonic() + command(['docker','stop','--time','7',item['container']],timeout=9) + state = inspect([item['container']])[0]['State'] + if state['Running'] or state['ExitCode'] != 0: + raise RuntimeError('API did not stop gracefully') + result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True} + (ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n') + return result + + +def restart(): + compose('restart','api','postgres','redis','minio','cleanup','lb',timeout=30) + compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40) + return wait_cluster(2,timeout=15) + + +def front_address(container): + networks = inspect([container])[0]['NetworkSettings']['Networks'] + return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front')) + + +def replace_one(container): + before = set(container_ids('api')) + if container not in before or len(before) != 2: + raise RuntimeError('Replacement expects exactly two running API containers') + old_address = front_address(container) + # Start-first ensures a genuinely different IP while the old address is still occupied. + compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api') + expanded = set(container_ids('api')) + added = expanded-before + if len(added) != 1 or not before.issubset(expanded): + raise RuntimeError('Start-first step unexpectedly changed an existing container') + replacement = added.pop() + wait_healthy(replacement) + new_address = front_address(replacement) + if new_address == old_address: + raise RuntimeError('Replacement did not get a different IP') + expected_three = {item['instance'] for item in replicas()} + if set(wait_cluster(3)) != expected_three: + raise RuntimeError('Load balancer did not discover all three live addresses') + command(['docker','stop','--time','7',container],timeout=10) + state = inspect([container])[0]['State'] + if state['ExitCode'] != 0: + raise RuntimeError('Old release did not terminate gracefully') + command(['docker','rm',container]) + compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api') + after = set(container_ids('api')) + if after != (before-{container}) | {replacement}: + raise RuntimeError('Sequential replacement changed the wrong containers') + expected = {item['instance'] for item in replicas()} + if set(wait_cluster(2)) != expected: + raise RuntimeError('Unexpected final backend aliases') + return {'old_container':container,'new_container':replacement, + 'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True, + 'instances_during_start_first':sorted(expected_three), + 'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec']) + parser.add_argument('arguments',nargs=argparse.REMAINDER) + args = parser.parse_args() + if args.action == 'wait': + print(json.dumps({'instances':wait_cluster()})) + elif args.action == 'snapshot': + print(json.dumps(snapshot(),indent=2)) + elif args.action == 'stop-one': + print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID']))) + elif args.action == 'exec': + ids = container_ids('api') + if not ids or not args.arguments: + raise RuntimeError('A live API container and command are required') + raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT)) + elif args.action == 'restart': + print(json.dumps({'instances':restart()})) + else: + compose('run','--rm','--no-deps','migrate') + print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..3985fa9c --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/cluster.py restart diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..ef14b25f --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +exec python3 scripts/cluster.py stop-one diff --git a/scripts/verify_attachments.py b/scripts/verify_attachments.py new file mode 100644 index 00000000..6f241d5c --- /dev/null +++ b/scripts/verify_attachments.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Grade4 attachment evidence with real S3 faults, process crash and grace cleanup.""" +import argparse +import base64 +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time +import urllib.request +import urllib.error +import urllib.parse +import uuid + +import cluster +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,Failure,check + + +def save(name,value): + (ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n') + print('PASS:',name,flush=True) + + +def internal(data,container=None): + container=container or cluster.container_ids('api')[0] + p=subprocess.run(['docker','exec','-i',container,'python','scripts/attachment_checks.py'],cwd=ROOT, + input=json.dumps(data),capture_output=True,text=True,check=True,timeout=30) + return json.loads(p.stdout) + + +def upload(c,actor,chat,raw,filename='sample.png'): + return c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+urllib.parse.quote(filename), + 201,actor=actor,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment') + + +def private_bucket(): + c=Client() + alice,bob,_,chat=c.bootstrap() + raw=(ROOT/'tests/fixtures/sample.png').read_bytes() + response=upload(c,alice,chat,raw) + att=response['json'] + key=internal({'action':'lookup','id':att['id']})['object_key'] + result=internal({'action':'security','key':key}) + check(result['anonymous_get']==403,'private object is accessible without S3 authentication') + check(result['create_bucket_with_api_credentials']==403 and result['list_other_prefix']==403,'API S3 credentials are too broad') + check(not result['console_reachable_from_api'] and not result['root_secret_mounted_in_api'],'root/console exposed to API') + before=internal({'action':'counts'}) + output=cluster.compose('run','--rm','--no-deps','minio-init') + check(internal({'action':'counts'})==before,'repeated bucket bootstrap changed data') + download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) + check(download['bytes']==raw,'bootstrap changed original bytes') + save('s3-security.json',{'result':'PASS',**result,'bootstrap_repeated_without_data_loss':True,'bootstrap_output':output}) + + +def upload_failover(): + cluster.wait_cluster() + c=Client() + alice,bob,_,chat=c.bootstrap() + raw=(ROOT/'tests/fixtures/sample.png').read_bytes() + response=upload(c,alice,chat,raw) + att=response['json'] + alias=response['headers']['X-Instance-Id'] + c.message(alice,chat,'',attachments=[att['id']]) + target=next(node for node in cluster.replicas() if node['instance']==alias) + started=time.monotonic() + try: + stopped=cluster.stop_one(alias) + deadline=started+10 + while True: + try: + download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) + break + except Failure: + check(time.monotonic()=1 and dry['locked']>=1,'dry-run/grace/lock selection failed') + check(internal({'action':'exists','key':orphan})['exists'],'dry-run deleted an object') + applied=internal({'action':'sweep','grace':60,'apply':True}) + check(applied['deleted']>=1,'orphan not removed after grace') + check(not internal({'action':'exists','key':orphan})['exists'],'crash orphan remained after cleanup') + check(internal({'action':'exists','key':locked})['exists'],'active upload lock was ignored') + check(internal({'action':'exists','key':fresh})['exists'],'fresh object was deleted') + repeat=internal({'action':'sweep','grace':60,'apply':True}) + check(repeat['deleted']==0,'repeated sweep was not idempotent') + check(c.call('GET','/api/v1/attachments/'+tracked['id']+'/content',actor=bob)['bytes']==raw,'tracked/linked file was deleted') + finally: + holder.stdin.write('\n') + holder.stdin.flush() + holder.stdin.close() + holder.wait(timeout=5) + released=internal({'action':'sweep','grace':60,'apply':True}) + check(not internal({'action':'exists','key':locked})['exists'],'unlocked old orphan not removed') + internal({'action':'delete','key':fresh}) + save('orphan-cleanup.json',{'result':'PASS','fault':'SIGKILL API after successful S3PUT, before metadata COMMIT', + 'crash_exit_code':observed_exit_code,'target_container':target['container'],'upload_acknowledged':False,'orphan_confirmed':True,'grace_wait_seconds':61, + 'dry_run':dry,'apply':applied,'repeat':repeat,'after_lock_release':released, + 'fresh_preserved':True,'tracked_and_linked_preserved':True,'active_lock_preserved':True}) + + +def s3_outage(): + c=Client(timeout=8) + alice,bob,_,chat=c.bootstrap() + raw=(ROOT/'tests/fixtures/sample.png').read_bytes() + att=upload(c,alice,chat,raw)['json'] + try: + cluster.compose('stop','-t','2','minio') + req=urllib.request.Request(c.base+'/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',data=raw,method='POST',headers={'Authorization':'Bearer '+alice['token'],'Content-Type':'image/png'}) + started=time.monotonic() + try: + response=urllib.request.urlopen(req,timeout=8) + except urllib.error.HTTPError as exc: + response=exc + with response: + status=response.code + elapsed=time.monotonic()-started + check(status in (502,503,504) and elapsed<8,'S3 outage was not bounded/safe') + finally: + cluster.compose('start','minio') + cluster.wait_healthy(cluster.container_ids('minio')[0]) + cluster.wait_cluster() + check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==raw,'original lost after S3 restart') + save('s3-outage.json',{'result':'PASS','upload_status_during_outage':status,'response_seconds':round(elapsed,4), + 'original_preserved_after_restart':True,'same_session_after_recovery':True}) + + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('mode',choices=['private','failover','interrupted','cleanup','outage','all'],default='all',nargs='?') + args=parser.parse_args() + modes={'private':private_bucket,'failover':upload_failover,'interrupted':interrupted_upload,'cleanup':crash_orphan,'outage':s3_outage} + if args.mode=='all': + for function in modes.values(): + function() + else: + modes[args.mode]() + + +if __name__=='__main__': + main() diff --git a/scripts/verify_cluster.py b/scripts/verify_cluster.py new file mode 100644 index 00000000..98653cf6 --- /dev/null +++ b/scripts/verify_cluster.py @@ -0,0 +1,443 @@ +#!/usr/bin/env python3 +"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts.""" +import argparse +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +import uuid + +import cluster + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client, Failure, check +from resilience import history + + +def save(name, value): + (ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n') + print('PASS:', name) + + +def percentiles(values): + values = sorted(values) + return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None + for name,p in [('p50',.50),('p95',.95),('p99',.99)]} + + +class Traffic: + def __init__(self, concurrency=4): + self.c = Client(timeout=6) + self.alice,self.bob,_,self.chat = self.c.bootstrap() + self.concurrency = concurrency + self.done = threading.Event() + self.lock = threading.Lock() + self.attempts = [] + self.confirmed = {} + self.prefix = uuid.uuid4().hex + + def worker(self, number): + client = Client(timeout=6) + index = 0 + while not self.done.is_set(): + text = self.prefix+'-'+str(number)+'-'+str(index) + key = self.prefix+'-'+str(number)+'-'+str(index) + started = time.monotonic() + try: + response = client.message(self.alice,self.chat,text,key=key) + row = response['json'] + attempt = {'started':started,'ended':time.monotonic(),'ok':True, + 'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']} + with self.lock: + self.confirmed[row['id']] = row + index += 1 + except Failure: + # Only idempotent message POST is retried, with its original body and key. + attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None} + time.sleep(0.02) + with self.lock: + self.attempts.append(attempt) + + def start(self): + self.started = time.monotonic() + self.pool = ThreadPoolExecutor(max_workers=self.concurrency) + self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)] + return self + + def stop(self): + self.done.set() + for future in self.futures: + future.result(timeout=10) + self.pool.shutdown() + self.ended = time.monotonic() + + def summary(self, start=None, end=None): + start = self.started if start is None else start + end = getattr(self,'ended',time.monotonic()) if end is None else end + with self.lock: + attempts = [a for a in self.attempts if start <= a['started'] < end] + success = [a for a in attempts if a['ok']] + duration = end-start + return {'seconds':round(duration,4),'concurrency':self.concurrency, + 'successful_requests':len(success),'failed_attempts':len(attempts)-len(success), + 'successful_rps':round(len(success)/duration,3), + 'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0, + 'successful_latency_ms':percentiles([a['latency'] for a in success]), + 'instances':sorted({a['instance'] for a in success})} + + def verify_data(self): + rows = history(self.c,self.bob,self.chat) + actual = {row['id']:row for row in rows} + check(len(actual)==len(rows),'duplicated message ids') + check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message') + check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed') + return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows), + 'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0} + + +def concurrent_replay(): + cluster.wait_cluster() + c = Client() + alice,bob,_,chat = c.bootstrap() + key = uuid.uuid4().hex + barrier = threading.Barrier(8) + def send(_): + local = Client() + barrier.wait() + return local.message(alice,chat,'Один результат через разные процессы',key=key) + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(send,range(8))) + instances = {r['headers']['X-Instance-Id'] for r in results} + check(len(instances)>=2,'concurrent retry did not reach different API processes') + check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages') + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice, + body={'text':'Другой body'},headers={'Idempotency-Key':key}) + check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates') + save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances), + 'responses_201':8,'messages_created':1,'different_body_status':409}) + + +def verify_persistence(): + before = cluster.snapshot() + result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'], + cwd=ROOT,check=True,capture_output=True,text=True,timeout=190) + after = cluster.snapshot() + roles = ['api','postgres','redis','minio','cleanup','lb'] + previous = {item['container']:item for item in before} + for item in after: + if item['service'] in roles: + old = previous[item['container']] + check(old['started_at']!=item['started_at'],'service did not restart: '+item['service']) + check(old['volumes']==item['volumes'],'volume identities changed') + (ROOT/'evidence/persistence.txt').write_text(result.stdout) + save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()}) + + +def public_failover(): + cluster.wait_cluster() + try: + result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'], + cwd=ROOT,check=True,capture_output=True,text=True,timeout=100) + (ROOT/'evidence/public-failover.txt').write_text(result.stdout) + save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(), + 'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())}) + finally: + cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90') + cluster.wait_cluster() + + +def reject_new_connection(survivor, address): + code = '''import json,socket,sys +try: + s=socket.create_connection((sys.argv[1],8080),timeout=1) +except OSError as exc: + print(json.dumps({'accepted':False,'error_type':type(exc).__name__})) +else: + s.close() + print(json.dumps({'accepted':True})) +''' + return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address])) + + +def failover_each(): + cluster.wait_cluster() + originals = cluster.replicas() + check(len(originals)==2,'two running replicas required') + traffic = Traffic().start() + phases = [] + try: + time.sleep(3) + baseline_end = time.monotonic() + baseline = traffic.summary(end=baseline_end) + for original in originals: + # Alias changes on restart, so resolve the current process of this exact container. + current = next(r for r in cluster.replicas() if r['container']==original['container']) + all_before = cluster.inspect([current['container']])[0] + address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front')) + survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container']) + read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1' + marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0] + start = time.monotonic() + try: + env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']} + subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10) + stop_returned = time.monotonic() + stopped = json.loads((ROOT/'evidence/last-stop.json').read_text()) + read_deadline = start+10 + while True: + try: + read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage') + check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage') + check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process') + read_done = time.monotonic() + break + except Failure: + check(time.monotonic()=start] + recent = [a for a in successful if a['started']>stop_returned] + recovery = min(a['ended'] for a in recent)-start if recent else None + after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None + check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start') + check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests') + phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4), + 'post_stop_write_seconds_from_stop_completion':round(after_stop,4), + 'read_during_outage':{'status':200,'confirmed_message_preserved':True, + 'seconds_from_action_start':round(read_done-start,4), + 'seconds_from_stop_completion':round(read_done-stop_returned,4)}, + 'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']}) + finally: + cluster.command(['docker','start',current['container']]) + cluster.wait_healthy(current['container']) + cluster.wait_cluster() + time.sleep(2) + finally: + traffic.stop() + save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases, + 'total':traffic.summary(),'data':traffic.verify_data(), + 'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2}) + + +def rolling_update(): + cluster.wait_cluster() + before = cluster.replicas() + check(len(before)==2,'rolling demonstration expects two replicas') + lb_before = next(item for item in cluster.snapshot() if item['service']=='lb') + target_release = 'lab4-'+uuid.uuid4().hex[:8] + traffic = Traffic().start() + replacements = [] + try: + time.sleep(3) + build_start = time.monotonic() + cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180) + build_end = time.monotonic() + migration_output = cluster.compose('run','--rm','--no-deps','migrate') + swap_start = time.monotonic() + for old in before: + step_start = time.monotonic() + result = cluster.replace_one(old['container']) + result['seconds'] = round(time.monotonic()-step_start,4) + replacements.append(result) + swap_end = time.monotonic() + time.sleep(3) + finally: + traffic.stop() + after = cluster.replicas() + lb_after = next(item for item in cluster.snapshot() if item['service']=='lb') + check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test') + check(all(r['release']==target_release for r in after),'new release not observed on all replicas') + check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement') + check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change') + save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release, + 'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True, + 'build_seconds':round(build_end-build_start,4),'migration_output':migration_output, + 'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end), + 'total':traffic.summary(),'data':traffic.verify_data(), + 'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'}) + + +def database_fixture(container, data): + code = '''import json,sys,psycopg +from psycopg import sql +from messenger.db import setting +p=json.load(sys.stdin) +with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c: + if p['action']=='install': + c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text']))) + c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function']))) + result={'installed':True} + elif p['action']=='remove': + c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger']))) + c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function']))) + result={'removed':True} + else: + result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]} +print(json.dumps(result)) +''' + response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT, + input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8) + return json.loads(response.stdout) + + +def graceful_shutdown(): + cluster.wait_cluster() + nodes = cluster.replicas() + target,survivor = nodes[0],nodes[1] + address = cluster.front_address(target['container']) + c = Client() + alice,bob,_,chat = c.bootstrap() + path = '/api/v1/chats/'+chat['id']+'/messages' + marker = 'drain_'+uuid.uuid4().hex + fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker} + database_fixture(survivor['container'],{**fixture,'action':'install'}) + code = '''import json,sys,urllib.request,urllib.error +p=json.load(sys.stdin) +r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']}) +try: + response=urllib.request.urlopen(r,timeout=6) +except urllib.error.HTTPError as exc: + response=exc +print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)})) +''' + request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex} + def in_flight(): + result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT, + input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8) + return json.loads(result.stdout) + pool = ThreadPoolExecutor(max_workers=1) + future = pool.submit(in_flight) + try: + deadline = time.monotonic()+3 + while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']: + check(time.monotonic()0 and item['nano_cpus']>0,'resource limit missing') + if item['service']!='lb': + check(not item['published_ports'],'service exposed a port bypassing LB') + check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas') + check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running') + save('infrastructure.json',{'result':'PASS','containers':state}) + + +def dependency_failures(): + c = Client(timeout=6) + actor = c.user('dependencies') + c.call('GET','/api/v1/users/me',actor=actor,schema='User') + result = {'result':'PASS'} + try: + cluster.compose('stop','-t','2','redis') + response = c.call('GET','/api/v1/users/me',actor=actor,schema='User') + check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed') + c.call('GET','/api/v1/users/me',401) + c.call('GET','/health/ready',schema='Health') + result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200} + finally: + cluster.compose('start','redis') + import urllib.request + import urllib.error + def status(path, token=None): + headers = {'Authorization':'Bearer '+token} if token else {} + request = urllib.request.Request(c.base+path,headers=headers) + started = time.monotonic() + try: + response = urllib.request.urlopen(request,timeout=6) + except urllib.error.HTTPError as error: + response = error + with response: + return response.code, round(time.monotonic()-started,4) + try: + cluster.compose('stop','-t','2','postgres') + code,elapsed = status('/api/v1/users/me',actor['token']) + check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request') + anonymous,_ = status('/api/v1/users/me') + check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage') + live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()] + check(all(code==200 for code in live),'API process liveness depends on DB') + result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous, + 'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'} + finally: + cluster.compose('start','postgres') + cluster.wait_cluster() + c.call('GET','/api/v1/users/me',actor=actor,schema='User') + result['same_session_after_recovery'] = True + save('dependency-failures.json',result) + + +def verify_nginx(): + state = cluster.snapshot() + lb = next(item for item in state if item['service']=='lb') + check(lb['image'].startswith('nginx:'),'load balancer must use Nginx') + cluster.command(['docker','exec',lb['container'],'nginx','-t']) + version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1']) + c = Client() + response = c.call('GET','/health/ready',schema='Health') + check(response['headers'].get('Server')=='nginx','public entry point is not Nginx') + check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas') + save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx', + 'config_syntax':'PASS','observed_instances':cluster.wait_cluster()}) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?') + args = parser.parse_args() + modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover, + 'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure} + if args.mode=='all': + for function in modes.values(): + function() + else: + modes[args.mode]() + + +if __name__ == '__main__': + main() diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/app.py b/src/messenger/app.py new file mode 100644 index 00000000..eb54d63a --- /dev/null +++ b/src/messenger/app.py @@ -0,0 +1,297 @@ +"""Contract-compatible synchronous REST API, with short bounded I/O in worker threads.""" +import json +import asyncio +import logging +import os +import re +from contextlib import asynccontextmanager +from uuid import uuid4 + +import psycopg +from fastapi import Depends, FastAPI, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from psycopg_pool import PoolTimeout, TooManyRequests +from redis.exceptions import RedisError +from starlette.exceptions import HTTPException +from starlette.concurrency import run_in_threadpool + +from .db import Database +from .errors import ApiError, invalid, not_found, unauthorized +from .models import CreateChat, CreateMessage, CreateUser, uuid_value +from .messages import store_message, validate_key, MESSAGE_FIELDS +from .object_store import ObjectStore, unavailable +from . import attachments +from urllib.parse import quote +from .pagination import decode_cursor, page, page_limit, public, stamp +from .security import HASHER, DUMMY_HASH, basic_credentials, bearer, check_password, digest, new_session + +LOG = logging.getLogger('messenger') +USER_FIELDS = 'id, username, display_name, created_at' + + +def error_response(request, error): + return JSONResponse({'error': {'code': error.code, 'message': error.message, + 'request_id': request.state.request_id}}, + status_code=error.status, headers=error.headers) + + +def authenticate(request: Request): + token = bearer(request.headers.get('Authorization', '')) + token_hash = digest(token) + with request.app.state.db.pool.connection() as conn: + row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()', + (token_hash,)).fetchone() + if not row: + raise unauthorized() + request.state.token_hash = token_hash + return row['user_id'] + + +def require_member(conn, chat_id, user_id): + found = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone() + if not found: + raise not_found() + + +def chat_public(conn, row): + item = public(row) + item['member_ids'] = [str(member['user_id']) for member in conn.execute( + 'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()] + return item + + +def create_app(): + @asynccontextmanager + async def lifespan(app): + app.state.db = Database() + app.state.storage = ObjectStore() + app.state.upload_slots = asyncio.Semaphore(6) + app.state.db.open() + try: + yield + finally: + app.state.storage.close() + app.state.db.close() + + app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + app.state.instance_id = uuid4().hex + app.state.release = os.environ.get('APP_RELEASE', 'lab4-v1') + + @app.middleware('http') + async def request_context(request: Request, call_next): + request.state.request_id = str(uuid4()) + try: + path = request.url.path + json_endpoint = path in ('/api/v1/users', '/api/v1/chats') or re.fullmatch(r'/api/v1/chats/[^/]+/messages', path) + if request.method == 'POST' and json_endpoint: + media = request.headers.get('Content-Type', '').split(';') + unsupported = media[0].strip().lower() != 'application/json' + for parameter in media[1:]: + name, _, value = parameter.strip().partition('=') + if name.lower() == 'charset' and value.strip('"').lower() not in ('utf-8', 'utf8'): + unsupported = True + if unsupported: + if path != '/api/v1/users': + await run_in_threadpool(authenticate, request) + raise ApiError(415, 'unsupported_media_type', 'Требуется application/json в UTF-8') + try: + json.loads((await request.body()).decode('utf-8')) + except (UnicodeError, json.JSONDecodeError): + raise invalid() from None + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc) + except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests): + response = error_response(request, ApiError(503, 'unavailable', 'Хранилище временно недоступно')) + except Exception as exc: + # Log exception class only: credentials, SQL and message content stay out of logs. + LOG.error('request_failed request_id=%s type=%s', request.state.request_id, type(exc).__name__) + response = error_response(request, ApiError(500, 'internal_error', 'Внутренняя ошибка')) + response.headers['X-Request-Id'] = request.state.request_id + response.headers['X-Instance-Id'] = app.state.instance_id + response.headers['X-App-Release'] = app.state.release + return response + + @app.exception_handler(ApiError) + async def api_error(request, exc): + return error_response(request, exc) + + @app.exception_handler(RequestValidationError) + async def validation_error(request, exc): + return error_response(request, invalid()) + + @app.exception_handler(HTTPException) + async def http_error(request, exc): + return error_response(request, not_found() if exc.status_code == 404 else invalid()) + + @app.get('/health/live') + def live(): + return {'status': 'ok'} + + @app.get('/health/ready') + def ready(request: Request): + try: + with request.app.state.db.pool.connection() as conn: + conn.execute('SELECT 1 FROM idempotency_records LIMIT 0') + conn.execute('SELECT 1 FROM attachments LIMIT 0') + except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn): + raise ApiError(503, 'unavailable', 'Схема хранилища ещё не готова') from None + request.app.state.storage.check() + return {'status': 'ok'} + + @app.post('/api/v1/users', status_code=201) + def register(body: CreateUser, request: Request): + password_hash = HASHER.hash(body.password) + try: + with request.app.state.db.pool.connection() as conn: + row = conn.execute( + f'INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING {USER_FIELDS}', + (uuid4(), body.username, body.display_name, password_hash)).fetchone() + except psycopg.errors.UniqueViolation: + raise ApiError(409, 'conflict', 'Имя пользователя занято') from None + return public(row) + + @app.post('/api/v1/auth/sessions', status_code=201) + async def create_session(request: Request): + # Run hashing and synchronous database calls outside the event loop. + username, password = basic_credentials(request.headers.get('Authorization', '')) + if await request.body(): + raise invalid() + return await run_in_threadpool(login, request, username, password) + + def login(request, username, password): + db = request.app.state.db + with db.pool.connection() as conn: + row = conn.execute('SELECT '+USER_FIELDS+', password_hash FROM users WHERE username=%s', (username,)).fetchone() + valid = check_password(row['password_hash'] if row else DUMMY_HASH, password) + if not row or not valid: + raise unauthorized(basic=True) + row.pop('password_hash') + token, expires_at = new_session(db.session_ttl) + with db.pool.connection() as conn: + conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)', + (digest(token), row['id'], expires_at)) + return {'token': token, 'token_type': 'Bearer', 'expires_at': stamp(expires_at), 'user': public(row)} + + @app.get('/api/v1/users/me') + def me(request: Request, user_id=Depends(authenticate)): + db = request.app.state.db + key = 'user:' + str(user_id) + try: + cached = db.cache.get(key) + if cached: + return json.loads(cached) + except (RedisError, ValueError): + pass + with db.pool.connection() as conn: + row = conn.execute('SELECT '+USER_FIELDS+' FROM users WHERE id=%s', (user_id,)).fetchone() + if not row: + raise unauthorized() + result = public(row) + try: + db.cache.set(key, json.dumps(result), ex=60) + except RedisError: + pass + return result + + @app.delete('/api/v1/auth/sessions/current', status_code=204) + def logout(request: Request, user_id=Depends(authenticate)): + with request.app.state.db.pool.connection() as conn: + conn.execute('DELETE FROM sessions WHERE token_hash=%s', (request.state.token_hash,)) + return Response(status_code=204) + + @app.post('/api/v1/chats', status_code=201) + def create_chat(body: CreateChat, request: Request, user_id=Depends(authenticate)): + if str(user_id) not in body.member_ids: + raise invalid() + try: + with request.app.state.db.pool.connection() as conn: + row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at', + (uuid4(), body.title)).fetchone() + # An unknown user raises AFTER the chat INSERT; transaction context rolls it all back. + with conn.cursor() as cur: + cur.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)', + [(row['id'], uuid_value(member)) for member in body.member_ids]) + result = chat_public(conn, row) + except psycopg.errors.ForeignKeyViolation: + raise invalid() from None + return result + + @app.get('/api/v1/chats') + def list_chats(request: Request, user_id=Depends(authenticate)): + limit = page_limit(request.query_params.get('limit', '50')) + scope = 'chats:' + str(user_id) + after = decode_cursor(scope, request.query_params.get('cursor')) + where = ' AND (c.created_at,c.id) > (%s,%s)' if after else '' + params = (user_id, *after, limit+1) if after else (user_id, limit+1) + with request.app.state.db.pool.connection() as conn: + rows = conn.execute('SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id ' + 'WHERE m.user_id=%s'+where+' ORDER BY c.created_at,c.id LIMIT %s', params).fetchall() + result = page(scope, rows, limit) + result['items'] = [chat_public(conn, row) for row in rows[:limit]] + return result + + @app.get('/api/v1/chats/{chat_id}') + def get_chat(chat_id: str, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + row = conn.execute('SELECT id,title,created_at FROM chats WHERE id=%s', (cid,)).fetchone() + return chat_public(conn, row) + + @app.post('/api/v1/chats/{chat_id}/messages', status_code=201) + def send_message(chat_id: str, body: CreateMessage, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + key = validate_key(request.headers.get('Idempotency-Key', '')) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + row = store_message(conn, user_id, cid, body.text, key, body.attachment_ids) + return public(row) + + @app.get('/api/v1/chats/{chat_id}/messages') + def history(chat_id: str, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + limit = page_limit(request.query_params.get('limit', '50')) + scope = 'messages:' + str(cid) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + after = decode_cursor(scope, request.query_params.get('cursor')) + where = ' AND (m.created_at,m.id) > (%s,%s)' if after else '' + params = (cid, *after, limit+1) if after else (cid, limit+1) + rows = conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.chat_id=%s'+where+ + ' ORDER BY m.created_at,m.id LIMIT %s', params).fetchall() + return page(scope, rows, limit) + + @app.post('/api/v1/chats/{chat_id}/attachments',status_code=201) + async def upload_attachment(chat_id: str,request: Request,filename: str='',user_id=Depends(authenticate)): + cid=uuid_value(chat_id) + def access(): + with request.app.state.db.pool.connection() as conn: + require_member(conn,cid,user_id) + await run_in_threadpool(access) + return await attachments.upload(request,user_id,cid,filename) + + @app.get('/api/v1/attachments/{attachment_id}') + def attachment_metadata(attachment_id: str,request: Request,user_id=Depends(authenticate)): + aid=uuid_value(attachment_id) + with request.app.state.db.pool.connection() as conn: + row=attachments.lookup(conn,aid,user_id) + return attachments.metadata(row) + + @app.get('/api/v1/attachments/{attachment_id}/content') + def attachment_content(attachment_id: str,request: Request,variant: str='original',user_id=Depends(authenticate)): + aid=uuid_value(attachment_id) + with request.app.state.db.pool.connection() as conn: + row=attachments.lookup(conn,aid,user_id) + if variant!='original': + raise invalid() + opened=request.app.state.storage.get(row['object_key']) + if opened.length!=row['size_bytes']: + opened.close() + raise unavailable() + return attachments.ObjectResponse(opened,media_type=row['content_type'],headers={ + 'Content-Length':str(row['size_bytes']), + 'Content-Disposition':"inline; filename*=UTF-8''"+quote(row['filename'],safe='')}) + + return app diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py new file mode 100644 index 00000000..21985ff1 --- /dev/null +++ b/src/messenger/attachments.py @@ -0,0 +1,176 @@ +"""Bounded binary upload, image validation and DB/S3 coordination.""" +import asyncio +import hashlib +import logging +import tempfile +import threading +import unicodedata +import warnings +from uuid import uuid4 + +import psycopg +from PIL import Image, PngImagePlugin, UnidentifiedImageError +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect +from starlette.responses import StreamingResponse + +from .errors import ApiError, invalid, not_found +from .pagination import public + +MAX_BYTES=10485760 +MAX_PIXELS=40000000 +Image.MAX_IMAGE_PIXELS=MAX_PIXELS +PngImagePlugin.MAX_TEXT_CHUNK=1048576 +PngImagePlugin.MAX_TEXT_MEMORY=1048576 +warnings.filterwarnings('error',category=Image.DecompressionBombWarning) +DECODER=threading.BoundedSemaphore(1) +LOG=logging.getLogger('messenger') +FIELDS='id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at' + + +def object_lock(conn,key,wait=True): + function='pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock' + row=conn.execute('SELECT '+function+'(hashtextextended(%s,0)) AS locked',(key,)).fetchone() + return True if wait else row['locked'] + + +def filename_value(value): + if not value or not value.strip() or len(value)>255: + raise invalid() + try: + value.encode('utf-8') + except UnicodeError: + raise invalid() from None + if any(c in '/\\' or unicodedata.category(c)=='Cc' for c in value): + raise invalid() + return value + + +def metadata(row): + result=public({key:value for key,value in row.items() if key!='object_key'}) + result.update(status='ready',variants=[],error_code=None) + return result + + +def validate_image(file,mime): + if not DECODER.acquire(timeout=3): + raise ApiError(503,'unavailable','Обработка изображений занята') + try: + file.seek(0) + with Image.open(file) as image: + actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(image.format) + if actual is None or actual!=mime: + raise ApiError(415,'unsupported_media_type','Ожидается PNG или JPEG с правильным MIME') + if image.width*image.height>MAX_PIXELS: + raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей') + image.verify() + file.seek(0) + with Image.open(file) as image: + image.load() + file.seek(0) + except (Image.DecompressionBombWarning,Image.DecompressionBombError): + raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей') from None + except (UnidentifiedImageError,OSError,SyntaxError,ValueError,EOFError): + raise ApiError(415,'unsupported_media_type','Повреждённое или неподдерживаемое изображение') from None + finally: + DECODER.release() + + +def compensate(db,store,key): + """A failed/unknown COMMIT is not proof of absence. Recheck under the same lock.""" + try: + with db.pool.connection() as conn: + if not object_lock(conn,key,wait=False): + return + exists=conn.execute('SELECT 1 FROM attachments WHERE object_key=%s',(key,)).fetchone() + if not exists: + store.delete(key) + except Exception as exc: + # A periodic, grace-protected janitor will retry after dependency recovery. + LOG.warning('attachment_compensation_deferred type=%s',type(exc).__name__) + + +def persist(db,store,user_id,chat_id,filename,mime,file,length,digest): + key='originals/'+uuid4().hex + try: + with db.pool.connection() as conn: + object_lock(conn,key) + store.put(key,file,length,mime) + row=conn.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key) ' + 'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING '+FIELDS, + (uuid4(),chat_id,user_id,filename,mime,length,digest,key)).fetchone() + return metadata(row) + except ApiError: + compensate(db,store,key) + raise + except psycopg.Error: + compensate(db,store,key) + raise ApiError(503,'unavailable','Не удалось сохранить вложение') from None + except BaseException: + compensate(db,store,key) + raise + + +async def upload(request,user_id,chat_id,filename): + filename=filename_value(filename) + mime=request.headers.get('Content-Type','').split(';')[0].strip().lower() + if mime not in ('image/png','image/jpeg'): + raise ApiError(415,'unsupported_media_type','Ожидается image/png или image/jpeg') + declared=request.headers.get('Content-Length') + if declared is not None: + try: + length=int(declared) + if length<0: + raise ValueError + except ValueError: + raise invalid() from None + if length>MAX_BYTES: + raise ApiError(413,'payload_too_large','Файл превышает 10 MiB') + try: + await asyncio.wait_for(request.app.state.upload_slots.acquire(),timeout=1) + except TimeoutError: + raise ApiError(503,'unavailable','Слишком много загрузок') from None + try: + with tempfile.TemporaryFile(mode='w+b') as file: + total=0 + digest=hashlib.sha256() + async def read_body(): + nonlocal total + async for chunk in request.stream(): + total+=len(chunk) + if total>MAX_BYTES: + raise ApiError(413,'payload_too_large','Файл превышает 10 MiB') + digest.update(chunk) + await run_in_threadpool(file.write,chunk) + try: + await asyncio.wait_for(read_body(),timeout=30) + except TimeoutError: + raise ApiError(400,'invalid_request','Время загрузки истекло') from None + except ClientDisconnect: + raise invalid() from None + await run_in_threadpool(validate_image,file,mime) + return await run_in_threadpool(persist,request.app.state.db,request.app.state.storage, + user_id,chat_id,filename,mime,file,total,digest.hexdigest()) + finally: + request.app.state.upload_slots.release() + + +def lookup(conn,attachment_id,user_id): + row=conn.execute('SELECT '+', '.join('a.'+field for field in (FIELDS+',object_key').split(','))+ + ' FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id ' + 'WHERE a.id=%s AND m.user_id=%s',(attachment_id,user_id)).fetchone() + if not row: + raise not_found() + return row + + +class ObjectResponse(StreamingResponse): + def __init__(self,opened,**kwargs): + self.opened=opened + super().__init__(opened.chunks(),**kwargs) + + async def __call__(self,scope,receive,send): + try: + await super().__call__(scope,receive,send) + finally: + self.opened.close() diff --git a/src/messenger/cleanup.py b/src/messenger/cleanup.py new file mode 100644 index 00000000..15c46010 --- /dev/null +++ b/src/messenger/cleanup.py @@ -0,0 +1,72 @@ +"""Grace-protected orphan cleanup; absence is verified in the primary DB under lock.""" +import argparse +from datetime import datetime,timezone,timedelta +import json +from pathlib import Path +import time + +from .attachments import object_lock +from .db import Database +from .object_store import ObjectStore + + +def sweep(db,store,grace_seconds=300,apply=False): + if grace_seconds<60: + raise ValueError('Grace must be at least60seconds') + cutoff=datetime.now(timezone.utc)-timedelta(seconds=grace_seconds) + result={'mode':'apply' if apply else 'dry-run','grace_seconds':grace_seconds, + 'listed':0,'fresh':0,'tracked':0,'locked':0,'orphans':0,'deleted':0} + for page in store.pages(): + for obj in page.get('Contents',[]): + result['listed']+=1 + if obj['LastModified']>=cutoff: + result['fresh']+=1 + continue + with db.pool.connection() as conn: + if not object_lock(conn,obj['Key'],wait=False): + result['locked']+=1 + continue + if conn.execute('SELECT 1 FROM attachments WHERE object_key=%s',(obj['Key'],)).fetchone(): + result['tracked']+=1 + continue + result['orphans']+=1 + if apply: + store.delete(obj['Key']) + result['deleted']+=1 + return result + + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('--grace-seconds',type=int,default=300) + parser.add_argument('--apply',action='store_true') + parser.add_argument('--dry-run',action='store_true') + parser.add_argument('--loop',action='store_true') + parser.add_argument('--interval',type=int,default=60) + args=parser.parse_args() + if args.grace_seconds<60 or args.interval<1 or (args.apply and args.dry_run): + parser.error('Use grace>=60, interval>=1 and either --apply or --dry-run') + db=Database() + store=ObjectStore() + db.open() + try: + while True: + try: + result=sweep(db,store,args.grace_seconds,args.apply) + print(json.dumps(result),flush=True) + except Exception as exc: + print(json.dumps({'error':type(exc).__name__,'deleted_unknown':False}),flush=True) + if not args.loop: + raise + finally: + Path('/tmp/cleanup-heartbeat').touch() + if not args.loop: + break + time.sleep(args.interval) + finally: + store.close() + db.close() + + +if __name__=='__main__': + main() diff --git a/src/messenger/db.py b/src/messenger/db.py new file mode 100644 index 00000000..27356ec1 --- /dev/null +++ b/src/messenger/db.py @@ -0,0 +1,42 @@ +"""Bounded dependencies; PostgreSQL is always the source of truth.""" +import os +from pathlib import Path + +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +from redis import Redis +from redis.backoff import NoBackoff +from redis.retry import Retry + + +def setting(name, default=None): + filename = os.environ.get(name + '_FILE') + value = Path(filename).read_text().strip() if filename else os.environ.get(name, default) + if value is None: + raise RuntimeError(f'Missing configuration: {name}') + return value + + +class Database: + def __init__(self): + self.session_ttl = int(setting('SESSION_TTL_SECONDS', '3600')) + if self.session_ttl < 1: + raise ValueError('SESSION_TTL_SECONDS must be positive') + self.pool = ConnectionPool( + setting('DATABASE_URL'), min_size=1, max_size=int(setting('POOL_MAX_SIZE', '8')), + timeout=2, max_waiting=16, open=False, + kwargs={'row_factory': dict_row, 'connect_timeout': 3, + 'options': '-c statement_timeout=3000 -c lock_timeout=2000 -c idle_in_transaction_session_timeout=5000'}, + ) + self.cache = Redis.from_url( + setting('CACHE_URL', 'redis://redis:6379/0'), decode_responses=True, + socket_connect_timeout=0.2, socket_timeout=0.2, max_connections=8, + retry=Retry(NoBackoff(), 0), + ) + + def open(self): + self.pool.open(wait=True, timeout=30) + + def close(self): + self.pool.close() + self.cache.close() diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 00000000..082390e7 --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,19 @@ +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def invalid(): + return ApiError(400, 'invalid_request', 'Неверный запрос') + + +def unauthorized(basic=False): + scheme = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"' + return ApiError(401, 'unauthorized', 'Требуется аутентификация', {'WWW-Authenticate': scheme}) + + +def not_found(): + return ApiError(404, 'not_found', 'Объект не найден') diff --git a/src/messenger/messages.py b/src/messenger/messages.py new file mode 100644 index 00000000..f4098a28 --- /dev/null +++ b/src/messenger/messages.py @@ -0,0 +1,52 @@ +"""One PostgreSQL transaction owns both a message and its retry result.""" +import hashlib +import json +import re +from uuid import uuid4 + +from .errors import ApiError, invalid, not_found +from .models import uuid_value + + +def validate_key(key): + if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key): + raise invalid() + return key + + +MESSAGE_FIELDS = 'm.id,m.chat_id,m.sender_id,m.text,m.created_at,ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY position) AS attachment_ids' + + +def store_message(conn, user_id, chat_id, text, key, attachment_ids=None): + attachment_ids=[] if attachment_ids is None else attachment_ids + payload={'text':text} + if attachment_ids: + payload['attachment_ids']=attachment_ids + canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + fingerprint = hashlib.sha256(canonical.encode('utf-8')).hexdigest() + params = (user_id, chat_id, key) + # A competing INSERT waits for the other transaction, then the next statement sees its result. + conn.execute("INSERT INTO idempotency_records(user_id,chat_id,key,request_hash,expires_at) " + "VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING", + (*params, fingerprint)) + record = conn.execute('SELECT request_hash,message_id,expires_at > clock_timestamp() AS active ' + 'FROM idempotency_records WHERE user_id=%s AND chat_id=%s AND key=%s FOR UPDATE', params).fetchone() + if record['message_id'] is not None and record['active']: + if record['request_hash'] != fingerprint: + raise ApiError(409, 'conflict', 'Ключ уже использован с другим сообщением') + return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s', + (record['message_id'],)).fetchone() + ids=[uuid_value(item) for item in attachment_ids] + if ids: + allowed=conn.execute('SELECT id FROM attachments WHERE chat_id=%s AND id=ANY(%s)',(chat_id,ids)).fetchall() + if len(allowed)!=len(ids): + raise not_found() + row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) ' + 'RETURNING id,chat_id,sender_id,text,created_at', (uuid4(), chat_id, user_id, text)).fetchone() + if ids: + with conn.cursor() as cur: + cur.executemany('INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)', + [(row['id'],aid,i) for i,aid in enumerate(ids)]) + conn.execute("UPDATE idempotency_records SET request_hash=%s,message_id=%s,expires_at=clock_timestamp()+interval '24 hours' " + 'WHERE user_id=%s AND chat_id=%s AND key=%s', (fingerprint, row['id'], *params)) + return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',(row['id'],)).fetchone() diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 00000000..e591e4ee --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,29 @@ +"""Versioned, atomic, repeatable migrations, serialized with an advisory lock.""" +import hashlib +from pathlib import Path + +import psycopg +from .db import setting + + +def main(): + directory = Path(__file__).resolve().parents[2] / 'migrations' + with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn: + conn.execute('SELECT pg_advisory_xact_lock(20261102)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())') + for path in sorted(directory.glob('*.sql')): + sql = path.read_text() + checksum = hashlib.sha256(sql.encode()).hexdigest() + found = conn.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone() + if found: + if found[0] != checksum: + raise RuntimeError('Applied migration checksum changed: ' + path.name) + continue + conn.execute(sql) + conn.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (path.name, checksum)) + print('Applied', path.name) + print('Migrations up to date') + + +if __name__ == '__main__': + main() diff --git a/src/messenger/models.py b/src/messenger/models.py new file mode 100644 index 00000000..85e68919 --- /dev/null +++ b/src/messenger/models.py @@ -0,0 +1,75 @@ +import re +from uuid import UUID +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from .errors import invalid + + +def uuid_value(value): + try: + # Require the UUID string representation, not integer/coercible values. + if not isinstance(value, str) or len(value) != 36: + raise ValueError + return UUID(value) + except (ValueError, TypeError, AttributeError): + raise invalid() from None + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra='forbid', strict=True) + + @field_validator('*') + @classmethod + def valid_strings(cls, value, info): + if isinstance(value, str): + if not value.strip() and info.field_name != 'text': + raise ValueError('Blank string') + # PostgreSQL cannot store NUL or unpaired Unicode surrogates. + if '\x00' in value: + raise ValueError('NUL is not supported') + value.encode('utf-8') + return value + + +class CreateUser(StrictModel): + username: str = Field(pattern=r'^[a-z0-9_]{3,32}$') + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator('username') + @classmethod + def username_ascii(cls, value): + if not re.fullmatch(r'[a-z0-9_]{3,32}', value): + raise ValueError('Invalid username') + return value + + +class CreateChat(StrictModel): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator('member_ids') + @classmethod + def members(cls, value): + ids = [uuid_value(item) for item in value] + if len(ids) != len(set(ids)): + raise ValueError('Duplicate member') + return [str(item) for item in ids] + + +class CreateMessage(StrictModel): + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list,max_length=10) + + @field_validator('attachment_ids') + @classmethod + def attachment_values(cls,value): + ids=[uuid_value(item) for item in value] + if len(ids)!=len(set(ids)): + raise ValueError('Duplicate attachment') + return [str(item) for item in ids] + + @model_validator(mode='after') + def nonempty_message(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError('Message must contain text or attachment') + return self diff --git a/src/messenger/object_store.py b/src/messenger/object_store.py new file mode 100644 index 00000000..59d3efe6 --- /dev/null +++ b/src/messenger/object_store.py @@ -0,0 +1,109 @@ +"""Small bounded S3 client. API credentials never contain root/admin permissions.""" +from contextlib import contextmanager +import threading + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from .db import setting +from .errors import ApiError + +PREFIX = 'originals/' + + +def unavailable(): + return ApiError(503,'unavailable','Хранилище файлов временно недоступно') + + +class OpenObject: + def __init__(self,body,release,length): + self.body=body + self.release=release + self.length=length + self.closed=False + self.close_lock=threading.Lock() + + def chunks(self): + try: + while True: + data=self.body.read(65536) + if not data: + break + yield data + finally: + self.close() + + def close(self): + with self.close_lock: + if self.closed: + return + self.closed=True + try: + self.body.close() + finally: + self.release() + + +class ObjectStore: + def __init__(self): + self.bucket=setting('S3_BUCKET','messenger-attachments') + self.gate=threading.BoundedSemaphore(8) + self.client=boto3.client('s3',endpoint_url=setting('S3_ENDPOINT','http://minio:9000'), + region_name=setting('S3_REGION','us-east-1'), + aws_access_key_id=setting('S3_ACCESS_KEY_ID'),aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), + config=Config(signature_version='s3v4',connect_timeout=1,read_timeout=3,max_pool_connections=8, + retries={'total_max_attempts':1,'mode':'standard'},s3={'addressing_style':'path'}, + request_checksum_calculation='when_required',response_checksum_validation='when_required')) + + @contextmanager + def operation(self): + if not self.gate.acquire(timeout=1): + raise unavailable() + try: + yield + except (BotoCoreError,ClientError): + raise unavailable() from None + finally: + self.gate.release() + + def check(self): + with self.operation(): + self.client.list_objects_v2(Bucket=self.bucket,Prefix=PREFIX,MaxKeys=1) + + def put(self,key,body,length,mime): + with self.operation(): + body.seek(0) + self.client.put_object(Bucket=self.bucket,Key=key,Body=body,ContentLength=length,ContentType=mime) + + def delete(self,key): + with self.operation(): + self.client.delete_object(Bucket=self.bucket,Key=key) + + def get(self,key): + if not self.gate.acquire(timeout=1): + raise unavailable() + try: + result=self.client.get_object(Bucket=self.bucket,Key=key) + return OpenObject(result['Body'],self.gate.release,result['ContentLength']) + except (BotoCoreError,ClientError): + self.gate.release() + raise unavailable() from None + except BaseException: + self.gate.release() + raise + + def pages(self): + token=None + while True: + params={'Bucket':self.bucket,'Prefix':PREFIX,'MaxKeys':1000} + if token: + params['ContinuationToken']=token + with self.operation(): + result=self.client.list_objects_v2(**params) + yield result + if not result.get('IsTruncated'): + break + token=result['NextContinuationToken'] + + def close(self): + self.client.close() diff --git a/src/messenger/pagination.py b/src/messenger/pagination.py new file mode 100644 index 00000000..14564c58 --- /dev/null +++ b/src/messenger/pagination.py @@ -0,0 +1,56 @@ +import base64 +import json +import re +from datetime import datetime, timezone +from uuid import UUID +from .errors import invalid +from .models import uuid_value + + +def stamp(value): + return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def encode_cursor(scope, row): + raw = json.dumps([scope, stamp(row['created_at']), str(row['id'])], separators=(',', ':')).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip('=') + + +def decode_cursor(scope, value): + if value is None: + return None + try: + if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', value): + raise ValueError + data = json.loads(base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True)) + if not isinstance(data, list) or len(data) != 3 or data[0] != scope: + raise ValueError + ts = datetime.fromisoformat(data[1].replace('Z', '+00:00')) + if ts.tzinfo is None: + raise ValueError + return ts, uuid_value(data[2]) + except (ValueError, TypeError, AttributeError, OverflowError): + raise invalid() from None + + +def page_limit(value): + if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100: + raise invalid() + return int(value) + + +def public(row): + def value(item): + if isinstance(item,datetime): + return stamp(item) + if isinstance(item,UUID): + return str(item) + if isinstance(item,list): + return [value(part) for part in item] + return item + return {key:value(item) for key,item in row.items()} + + +def page(scope, rows, limit): + return {'items': [public(row) for row in rows[:limit]], + 'next_cursor': encode_cursor(scope, rows[limit-1]) if len(rows) > limit else None} diff --git a/src/messenger/security.py b/src/messenger/security.py new file mode 100644 index 00000000..b82c6b0c --- /dev/null +++ b/src/messenger/security.py @@ -0,0 +1,49 @@ +import base64 +import hashlib +import re +import secrets +from datetime import datetime, timedelta, timezone +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError, InvalidHashError +from .errors import unauthorized + +HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +DUMMY_HASH = HASHER.hash('synthetic-dummy-password') + + +def digest(token): + return hashlib.sha256(token.encode()).hexdigest() + + +def basic_credentials(header): + try: + scheme, encoded = header.split(' ', 1) + if scheme.lower() != 'basic': + raise ValueError + username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + if not re.fullmatch(r'[a-z0-9_]{3,32}', username) or not 12 <= len(password) <= 128: + raise ValueError + return username, password + except (ValueError, UnicodeError): + raise unauthorized(basic=True) from None + + +def check_password(encoded, password): + try: + return HASHER.verify(encoded, password) + except (VerificationError, InvalidHashError): + return False + + +def bearer(header): + try: + scheme, token = header.split(' ', 1) + if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token): + raise ValueError + return token + except ValueError: + raise unauthorized() from None + + +def new_session(ttl): + return secrets.token_urlsafe(32), datetime.now(timezone.utc) + timedelta(seconds=ttl) diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..6342a6ae --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,141 @@ +"""Integration tests: real PostgreSQL and Redis, isolated synthetic identities.""" +import base64 +import json +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +import pytest +from fastapi.testclient import TestClient +from messenger.app import create_app + +PASSWORD = 'Тестовый:пароль🔐' * 8 + + +@pytest.fixture +def client(): + with TestClient(create_app()) as c: + yield c + + +def user(c): + body = dict(username='u_' + uuid.uuid4().hex[:24], display_name='Алиса 👋', password=PASSWORD) + r = c.post('/api/v1/users', json=body) + assert r.status_code == 201, r.text + return r.json(), body + + +def login(c, body): + credentials = base64.b64encode((body['username'] + ':' + body['password']).encode()).decode() + r = c.post('/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credentials}) + assert r.status_code == 201, r.text + return {'Authorization': 'Bearer ' + r.json()['token']}, r.json() + + +def test_auth_unicode_logout_and_cache(client): + u, body = user(client) + h, session = login(client, body) + assert session['user'] == u + assert len(session['token']) >= 43 + key = 'user:' + u['id'] + client.app.state.db.cache.delete(key) + for _ in range(2): + assert client.get('/api/v1/users/me', headers=h).json() == u + cached = json.loads(client.app.state.db.cache.get(key)) + assert cached == u + assert 0 < client.app.state.db.cache.ttl(key) <= 60 + assert client.delete('/api/v1/auth/sessions/current', headers=h).status_code == 204 + r = client.get('/api/v1/users/me', headers=h) + assert r.status_code == 401 + assert r.headers['www-authenticate'] == 'Bearer realm="messenger"' + assert r.json()['error']['request_id'] == r.headers['x-request-id'] + + +def test_session_expiry(monkeypatch): + monkeypatch.setenv('SESSION_TTL_SECONDS', '2') + with TestClient(create_app()) as client: + _, body = user(client) + h, _ = login(client, body) + assert client.get('/api/v1/users/me', headers=h).status_code == 200 + time.sleep(2.1) + assert client.get('/api/v1/users/me', headers=h).status_code == 401 + + +def test_registration_race(client): + body = dict(username='race_' + uuid.uuid4().hex[:22], display_name='Race', password=PASSWORD) + with ThreadPoolExecutor(max_workers=6) as executor: + codes = list(executor.map(lambda _: client.post('/api/v1/users', json=body).status_code, range(6))) + assert sorted(codes) == [201, 409, 409, 409, 409, 409] + + +def test_chat_rollback_after_insert(client): + u, body = user(client) + h, _ = login(client, body) + title = 'rollback_' + uuid.uuid4().hex + r = client.post('/api/v1/chats', headers=h, json={'title': title, 'member_ids': [u['id'], str(uuid.uuid4())]}) + assert r.status_code == 400 + with client.app.state.db.pool.connection() as conn: + assert conn.execute('SELECT count(*) AS n FROM chats WHERE title=%s', (title,)).fetchone()['n'] == 0 + assert conn.execute('SELECT count(*) AS n FROM chat_members m LEFT JOIN chats c ON c.id=m.chat_id WHERE c.id IS NULL').fetchone()['n'] == 0 + + +def test_cursor_access_and_strict_validation(client): + a, ab = user(client) + b, bb = user(client) + ah, _ = login(client, ab) + bh, _ = login(client, bb) + chats = [client.post('/api/v1/chats', headers=ah, json={'title':'Чат','member_ids':[a['id']]}).json() for _ in range(2)] + path = '/api/v1/chats/' + chats[0]['id'] + for text in ['Один 👋', 'Два', 'Три']: + assert client.post(path+'/messages', headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, json={'text':text}).status_code == 201 + p = client.get(path+'/messages?limit=2', headers=ah).json() + assert [m['text'] for m in p['items']] == ['Один 👋','Два'] + tail = client.get(path+'/messages', headers=ah, params={'cursor':p['next_cursor'],'limit':1}).json() + assert [m['text'] for m in tail['items']] == ['Три'] + assert tail['next_cursor'] is None + r = client.get('/api/v1/chats/'+chats[1]['id']+'/messages',headers=ah, params={'cursor':p['next_cursor']}) + assert r.status_code == 400 + assert client.get(path, headers=bh).status_code == 404 + for payload in [{'text':123}, {'text':' '}, {'text':'ok','sender_id':b['id']}, {'text':'x'*4001}]: + assert client.post(path+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json=payload).status_code == 400 + for query in ['limit=0','limit=101','limit=1.0','cursor=bad','cursor=']: + assert client.get(path+'/messages?'+query,headers=ah).status_code == 400 + for payload in [{'username':123,'display_name':'a','password':PASSWORD}, {'username':'valid','display_name':' ','password':PASSWORD}]: + assert client.post('/api/v1/users',json=payload).status_code == 400 + assert client.post('/api/v1/users',content='{',headers={'Content-Type':'application/json'}).status_code == 400 + + +def test_session_storage_and_migrations(client): + from messenger.migrate import main as migrate + u, body = user(client) + _, session = login(client, body) + db = client.app.state.db + def snapshot(): + with db.pool.connection() as conn: + return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n'] + for table in ['users','chats','chat_members','messages','sessions','schema_migrations','idempotency_records']} + before = snapshot() + migrate() + migrate() + assert snapshot() == before + with db.pool.connection() as conn: + row = conn.execute('SELECT token_hash,expires_at FROM sessions WHERE user_id=%s', (u['id'],)).fetchone() + assert row['token_hash'] != session['token'] + assert len(row['token_hash']) == 64 + stored = conn.execute('SELECT password_hash FROM users WHERE id=%s', (u['id'],)).fetchone()['password_hash'] + assert stored.startswith('$argon2id$') + assert stored != body['password'] + + +def test_json_media_type_and_utf8(client): + body = dict(username='media_' + uuid.uuid4().hex[:24], display_name='UTF8', password=PASSWORD) + raw = json.dumps(body).encode() + for headers in [{}, {'Content-Type':'text/plain'}, {'Content-Type':'application/xml'}]: + r = client.post('/api/v1/users',content=raw,headers=headers) + assert r.status_code == 415 + assert r.json()['error']['code'] == 'unsupported_media_type' + for encoding in ['utf-16', 'utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be']: + r = client.post('/api/v1/users',content=json.dumps(body).encode(encoding),headers={'Content-Type':'application/json'}) + assert r.status_code == 400, encoding + r = client.post('/api/v1/chats',content=b'{}',headers={'Content-Type':'text/plain'}) + assert r.status_code == 401 diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..4b8be1ca --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,167 @@ +"""Real PostgreSQL + private S3 integration; synthetic images only.""" +import hashlib +import io +import struct +import uuid +import zlib +from PIL import Image +from psycopg import sql +from test_api import client, user, login + + +def image_bytes(kind='PNG'): + stream = io.BytesIO() + Image.new('RGB',(320,200),(45,110,190)).save(stream,format=kind) + return stream.getvalue() + + +def actors(c): + a,ab=user(c) + b,bb=user(c) + e,eb=user(c) + ah,_=login(c,ab) + bh,_=login(c,bb) + eh,_=login(c,eb) + chat=c.post('/api/v1/chats',headers=ah,json={'title':'Файлы','member_ids':[a['id'],b['id']]}).json() + return ah,bh,eh,chat + + +def upload(c,headers,chat,raw=None,mime='image/png',filename='Фото.png'): + raw=image_bytes() if raw is None else raw + return c.post('/api/v1/chats/'+chat['id']+'/attachments',params={'filename':filename},content=raw, + headers={**headers,'Content-Type':mime}) + + +def test_original_png_jpeg_and_message(client): + ah,bh,eh,chat=actors(client) + for kind,mime in [('PNG','image/png'),('JPEG','image/jpeg')]: + raw=image_bytes(kind) + r=upload(client,ah,chat,raw,mime) + assert r.status_code==201 + att=r.json() + assert att['status']=='ready' and att['variants']==[] and att['error_code'] is None + assert att['sha256']==hashlib.sha256(raw).hexdigest() and att['size_bytes']==len(raw) + assert 'object_key' not in att and 'bucket' not in att + path='/api/v1/attachments/'+att['id'] + assert client.get(path,headers=bh).json()==att + content=client.get(path+'/content',headers=bh) + assert content.status_code==200 and content.content==raw + assert content.headers['Content-Type']==mime + for suffix in ['', '/content']: + assert client.get(path+suffix,headers=eh).status_code==404 + assert client.get(path+'/content?variant=thumbnail',headers=bh).status_code==400 + response=client.post('/api/v1/chats/'+chat['id']+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, + json={'text':'','attachment_ids':[att['id']]}) + assert response.status_code==201 + assert response.json()['attachment_ids']==[att['id']] + + +def test_invalid_media_corruption_limits_and_filename(client): + ah,_,_,chat=actors(client) + cases=[(b'not an image','image/png',415),(image_bytes(),'image/jpeg',415), + (image_bytes('JPEG'),'image/png',415),(image_bytes()[:40],'image/png',415), + (image_bytes(),'text/plain',415),(b'x'*(10485760+1),'image/png',413)] + for raw,mime,expected in cases: + response=upload(client,ah,chat,raw,mime) + assert response.status_code==expected + assert response.json()['error']['request_id']==response.headers['X-Request-Id'] + # Valid PNG header advertises >40Mp; reject before attempting decompression. + header=struct.pack('!IIBBBBB',10001,4000,8,2,0,0,0) + chunk=lambda tag,data: struct.pack('!I',len(data))+tag+data+struct.pack('!I',zlib.crc32(tag+data)&0xffffffff) + bomb=b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',header)+chunk(b'IDAT',zlib.compress(b''))+chunk(b'IEND',b'') + assert upload(client,ah,chat,bomb).status_code==413 + for filename in ['../x.png','a\\b.png','bad\x00.png','bad\x1f.png','bad\x7f.png',' ','x'*256]: + assert upload(client,ah,chat,filename=filename).status_code==400 + + +def test_foreign_attachment_binding_and_normalized_replay(client): + ah,bh,eh,chat=actors(client) + att=upload(client,ah,chat).json() + other=client.post('/api/v1/chats',headers=bh,json={'title':'Другой чат','member_ids':[chat['member_ids'][0],chat['member_ids'][1]]}).json() + path='/api/v1/chats/'+other['id']+'/messages' + for aid in [att['id'],str(uuid.uuid4())]: + response=client.post(path,headers={**bh,'Idempotency-Key':uuid.uuid4().hex},json={'text':'test','attachment_ids':[aid]}) + assert response.status_code==404 + assert upload(client,eh,chat).status_code==404 + h={**ah,'Idempotency-Key':uuid.uuid4().hex} + path='/api/v1/chats/'+chat['id']+'/messages' + first=client.post(path,headers=h,json={'text':'same'}) + replay=client.post(path,headers=h,json={'attachment_ids':[],'text':'same'}) + assert first.status_code==replay.status_code==201 and first.json()==replay.json() + conflict=client.post(path,headers=h,json={'text':'same','attachment_ids':[att['id']]}) + assert conflict.status_code==409 + duplicates=client.post(path,headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[att['id'],att['id']]}) + assert duplicates.status_code==400 + + +def test_compensate_s3_when_metadata_insert_fails(client): + ah,_,_,chat=actors(client) + db=client.app.state.db + store=client.app.state.storage + before={item['Key'] for page in store.pages() for item in page.get('Contents',[])} + function='fail_attachment_'+uuid.uuid4().hex + trigger='fail_attachment_'+uuid.uuid4().hex + filename='fault_'+uuid.uuid4().hex+'.png' + with db.pool.connection() as conn: + conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename={} THEN RAISE EXCEPTION 'synthetic metadata failure'; END IF; RETURN NEW; END $$").format(sql.Identifier(function),sql.Literal(filename))) + conn.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function))) + try: + response=upload(client,ah,chat,filename=filename) + assert response.status_code==503 + with db.pool.connection() as conn: + assert conn.execute('SELECT count(*) AS n FROM attachments WHERE filename=%s',(filename,)).fetchone()['n']==0 + after={item['Key'] for page in store.pages() for item in page.get('Contents',[])} + assert after==before + finally: + with db.pool.connection() as conn: + conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON attachments').format(sql.Identifier(trigger))) + conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function))) + + +def test_compensation_preserves_committed_object(client): + from messenger.attachments import compensate + ah,_,_,chat=actors(client) + att=upload(client,ah,chat).json() + with client.app.state.db.pool.connection() as conn: + key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key'] + # A retry after an unknown COMMIT result must never delete an already tracked object. + compensate(client.app.state.db,client.app.state.storage,key) + content=client.get('/api/v1/attachments/'+att['id']+'/content',headers=ah) + assert content.status_code==200 and content.content==image_bytes() + + +def test_stream_close_even_if_headers_cannot_be_sent(client): + import asyncio + import pytest + from messenger.attachments import ObjectResponse + ah,_,_,chat=actors(client) + att=upload(client,ah,chat).json() + with client.app.state.db.pool.connection() as conn: + key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key'] + store=client.app.state.storage + opened=store.get(key) + response=ObjectResponse(opened,media_type='image/png') + async def fail_send(message): + raise OSError('synthetic disconnected transport') + async def receive(): + return {'type':'http.disconnect'} + with pytest.raises(Exception): + asyncio.run(response({'type':'http','asgi':{'spec_version':'2.4'}},receive,fail_send)) + assert opened.closed + acquired=0 + try: + for _ in range(8): + assert store.gate.acquire(blocking=False) + acquired+=1 + finally: + for _ in range(acquired): + store.gate.release() + + +def test_chunked_upload_is_limited_without_content_length(client): + ah,_,_,chat=actors(client) + chunks=iter([b'x'*1048576]*10+[b'x']) + response=client.post('/api/v1/chats/'+chat['id']+'/attachments?filename=chunked.png', + content=chunks,headers={**ah,'Content-Type':'image/png'}) + assert response.status_code==413 + assert response.json()['error']['code']=='payload_too_large' diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..826d3dc1 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,119 @@ +"""Regressions for lab3 shared message results, real PostgreSQL (no mocks).""" +from concurrent.futures import ThreadPoolExecutor +import uuid + +from psycopg import sql +from test_api import client, user, login + + +def setup_chat(c, extra=None): + u, body = user(c) + headers, _ = login(c, body) + members = [u['id']] + (extra or []) + chat = c.post('/api/v1/chats', headers=headers, json={'title':'Идемпотентность', 'member_ids':members}).json() + return u, headers, '/api/v1/chats/'+chat['id']+'/messages' + + +def test_instance_alias_on_success_and_errors(client): + responses = [client.get('/health/live'), client.get('/health/ready'), client.get('/api/v1/users/me'), + client.get('/unknown'), client.post('/api/v1/users', content='{', headers={'Content-Type':'application/json'})] + ids = [r.headers.get('X-Instance-Id') for r in responses] + assert ids[0] and len(set(ids)) == 1 + assert all(r.headers.get('X-Request-Id') for r in responses) + + +def test_key_required_replay_and_conflict(client): + _, headers, path = setup_chat(client) + assert client.post(path, headers=headers, json={'text':'same'}).status_code == 400 + for key in ['', 'a b', 'bad/', 'x'*129]: + assert client.post(path, headers={**headers,'Idempotency-Key':key},json={'text':'same'}).status_code == 400 + h = {**headers, 'Idempotency-Key':'valid:._-123'} + first = client.post(path, headers=h, json={'text':'same'}) + assert first.status_code == 201 + replay = client.post(path, headers={**h,'Content-Type':'application/json'},content=' { "text" : "same" } ') + assert replay.status_code == 201 and replay.json() == first.json() + assert client.post(path, headers=h, json={'text':'other'}).status_code == 409 + assert len(client.get(path,headers=headers).json()['items']) == 1 + + +def test_concurrent_key_produces_one_message(client): + _, headers, path = setup_chat(client) + h = {**headers, 'Idempotency-Key':uuid.uuid4().hex} + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(lambda _: client.post(path,headers=h,json={'text':'concurrent'}), range(16))) + assert [r.status_code for r in results] == [201]*16 + assert all(r.json() == results[0].json() for r in results) + assert client.get(path,headers=headers).json()['items'] == [results[0].json()] + + +def test_scope_and_replay_rechecks_authorization(client): + b, bb = user(client) + bh, _ = login(client, bb) + a, ah, path = setup_chat(client, [b['id']]) + key = 'same-in-independent-scopes' + first = client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).json() + second = client.post(path,headers={**bh,'Idempotency-Key':key},json={'text':'b'}) + assert second.status_code == 201 and second.json()['id'] != first['id'] + chat = client.post('/api/v1/chats',headers=ah,json={'title':'Другая область','member_ids':[a['id']]}).json() + other_path = '/api/v1/chats/'+chat['id']+'/messages' + third = client.post(other_path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}) + assert third.status_code == 201 and third.json()['id'] != first['id'] + outsider, ob = user(client) + oh, _ = login(client, ob) + assert client.post(path,headers={**oh,'Idempotency-Key':key},json={'text':'a'}).status_code == 404 + assert client.delete('/api/v1/auth/sessions/current',headers=ah).status_code == 204 + assert client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).status_code == 401 + + +def test_expired_key_can_be_reused(client): + u, h, path = setup_chat(client) + key = uuid.uuid4().hex + first = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'first'}).json() + with client.app.state.db.pool.connection() as conn: + ttl = conn.execute('SELECT extract(epoch FROM expires_at-clock_timestamp()) AS seconds FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['seconds'] + assert 86390 <= ttl <= 86400 + conn.execute("UPDATE idempotency_records SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=%s AND key=%s", (u['id'],key)) + second = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'second'}) + assert second.status_code == 201 and second.json()['id'] != first['id'] + assert len(client.get(path,headers=h).json()['items']) == 2 + + +def test_rollback_side_effect_and_key_on_database_error(client): + u, h, path = setup_chat(client) + key = uuid.uuid4().hex + text = 'fault_' + uuid.uuid4().hex + function = 'fault_' + uuid.uuid4().hex + trigger = 'trigger_' + uuid.uuid4().hex + db = client.app.state.db + # AFTER INSERT fails after the side effect. This is a DB-only test fixture, not a production hook. + with db.pool.connection() as conn: + conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN RAISE EXCEPTION 'synthetic insert fault'; END IF; RETURN NEW; END $$").format(sql.Identifier(function), sql.Literal(text))) + conn.execute(sql.SQL('CREATE TRIGGER {} AFTER INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function))) + try: + failed = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text}) + assert failed.status_code == 500 + assert failed.headers['X-Instance-Id'] + with db.pool.connection() as conn: + assert conn.execute('SELECT count(*) AS n FROM messages WHERE text=%s',(text,)).fetchone()['n'] == 0 + assert conn.execute('SELECT count(*) AS n FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['n'] == 0 + finally: + with db.pool.connection() as conn: + conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(trigger))) + conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function))) + retried = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text}) + assert retried.status_code == 201 + assert client.get(path,headers=h).json()['items'] == [retried.json()] + + +def test_ready_requires_compatible_schema(client): + temporary = 'temporarily_unavailable_' + uuid.uuid4().hex + with client.app.state.db.pool.connection() as conn: + conn.execute(sql.SQL('ALTER TABLE idempotency_records RENAME TO {}').format(sql.Identifier(temporary))) + try: + r = client.get('/health/ready') + assert r.status_code == 503 + assert r.json()['error']['code'] == 'unavailable' + assert client.get('/health/live').status_code == 200 + finally: + with client.app.state.db.pool.connection() as conn: + conn.execute(sql.SQL('ALTER TABLE {} RENAME TO idempotency_records').format(sql.Identifier(temporary)))