ЛР 4 — БПИ2303 — Лукьянов — на 4 #100

Open
xabenskie wants to merge 1 commits from xabenskie/mtusi2026lab4:solution/bpi2303-lukyanov into main
65 changed files with 5025 additions and 33 deletions
+3
View File
@@ -18,3 +18,6 @@ __pycache__
node_modules
evidence
.DS_Store
.superpowers
.pytest_cache
+5 -19
View File
@@ -1,20 +1,6 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
# Created by scripts/bootstrap.sh; .env and secrets/ are never committed.
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
# Compose supplies DATABASE_URL_FILE, CACHE_URL_FILE and S3 credentials *_FILE.
# S3_ENDPOINT=http://minio:9000, S3_REGION=us-east-1, S3_BUCKET=messenger-attachments.
# Contract limits:10MiB and40M pixels. Six uploads and one decoder per API.
# Cleanup uses grace300s, interval60s. Only Nginx publishes127.0.0.1:8080.
+26
View File
@@ -0,0 +1,26 @@
name: Application acceptance
on: [push, pull_request]
permissions:
contents: read
jobs:
cluster:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.13'
- run: make up
- run: make check test test-student
- run: make test-fresh
- run: make test-cluster
- run: make test-attachments
- run: make uploads
- uses: actions/upload-artifact@v4
if: always()
with:
name: lab4-evidence
path: evidence/
- if: always()
run: docker compose down
+18
View File
@@ -0,0 +1,18 @@
FROM python:3.13.5-slim
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src
WORKDIR /app
COPY requirements.txt requirements-dev.txt requirements.lock ./
RUN pip install --no-cache-dir -r requirements.lock -r requirements.txt && useradd --uid 10001 --create-home messenger
COPY src/ src/
COPY migrations/ migrations/
COPY student-tests/ student-tests/
COPY scripts/ scripts/
COPY tests/ tests/
COPY contracts/ contracts/
COPY lab.json Makefile README.md COURSE.md CONTRIBUTING.md REPORT.md ./
ARG APP_RELEASE=lab4-v1
ENV APP_RELEASE=${APP_RELEASE}
LABEL org.opencontainers.image.version=${APP_RELEASE}
USER messenger
EXPOSE 8080
CMD ["uvicorn", "messenger.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5", "--no-access-log"]
+39
View File
@@ -28,3 +28,42 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: bootstrap up down test-student test-cluster rolling
bootstrap:
./scripts/bootstrap.sh
up: bootstrap
docker compose build api minio minio-init
docker compose up -d --wait --wait-timeout 120
$(PYTHON) scripts/cluster.py wait
down:
docker compose down
test-student:
$(PYTHON) scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
test-cluster:
$(PYTHON) scripts/verify_cluster.py
rolling:
$(PYTHON) scripts/cluster.py rolling
.PHONY: test-fresh load-evidence verify-persistence verify-failover
test-fresh:
./scripts/check_fresh_migrations.sh
load-evidence:
$(PYTHON) scripts/capture_load.py
verify-persistence:
$(PYTHON) scripts/verify_cluster.py persistence
verify-failover:
$(PYTHON) scripts/verify_cluster.py public-failover
$(PYTHON) scripts/verify_cluster.py failover
.PHONY: validate-container
validate-container:
$(PYTHON) scripts/cluster.py exec python scripts/validate_openapi.py
.PHONY: test-attachments uploads cleanup-dry-run
test-attachments:
$(PYTHON) scripts/verify_attachments.py
uploads:
$(PYTHON) scripts/capture_uploads.py
cleanup-dry-run:
docker compose run --rm --no-deps cleanup python -m messenger.cleanup --dry-run
+126 -14
View File
@@ -1,37 +1,149 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Автор, группа: Лукьянов К.Р. БПИ2303
- Целевая оценка; выбранный трек 5, если нужен: **4** (трек 5 не заявлен)
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Использованные источники, библиотеки и помощь ИИ: FastAPI, Pydantic v2, Uvicorn, Psycopg 3, redis-py, argon2-cffi, boto3/botocore, Pillow, pytest; документация [S3 uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html), [Pillow Image](https://pillow.readthedocs.io/en/stable/reference/Image.html), [Boto3 configuration](https://docs.aws.amazon.com/boto3/latest/guide/configuration.html), [MinIO IAM](https://github.com/minio/minio/blob/master/docs/multi-user/README.md), [Nginx upstream](https://nginx.org/en/docs/http/ngx_http_upstream_module.html). Реализация выполнена совместно мной и Codex: часть кода, тестов и отчёта написана с помощью Codex, результат проверен запуском тестов, описанных ниже.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Prerequisites: Docker с Compose v2, Python ≥ 3.9 на хосте для стандартных тестов/оркестрации, Make, свободный localhost:8080. Проверено на Docker Desktop 29.8.2, macOS/Apple M4 Pro. Go и Python-зависимости приложения устанавливаются внутри сборочных контейнеров. Для первой сборки нужен доступ к GitHub и Go/Python module repositories.
```sh
git clone <fork-url> && cd <каталог-fork-лабораторной-4>
make up # secrets → сборка API/MinIO/mc → migrate/init → api×2/cleanup/Nginx
curl -fsS http://localhost:8080/health/ready
docker compose ps --all # API×2/PG/Redis/MinIO/cleanup/LB работают; migrate и minio-init — Exited(0)
make check
make validate-container
make test # публичный smoke lab4 через Nginx
make test-student # 21 собственный интеграционный тест
make test-fresh # миграции 001–003, новый bucket и тесты на отдельных пустых volumes
make test-attachments # private bucket, upload→отказ API→download, обрыв, crash+cleanup, отказ S3
make uploads # 10 одновременных загрузок с измерением пика RAM
make test-cluster # сохранённые проверки общей сессии/idempotency/restart/failover/SIGTERM/rolling
```
`make up` выполняет bootstrap, `docker compose build api minio minio-init`, затем `up --wait` и ожидание двух API-alias через LB. При повторной сборке Go/Python-слои используются из кэша. Сценарии отказов и замер памяти запускать по одному: они останавливают/пересоздают контейнеры. `make uploads` пересоздаёт только API, чтобы сбросить kernel memory high-water marks; volumes и данные сохраняются.
**Почему MinIO собирается из исходников:** проверенные release-образы в Docker Hub/Quay недоступны, официальный архив бинарников ответил 410. В `infra/minio/Dockerfile` собираются официальные теги и проверяется точный commit: MinIO `RELEASE.2025-04-22T22-12-26Z` / `0d7408fc9969caf07de6a8c3a84f9fbb10a6739e`, mc `RELEASE.2025-04-16T18-13-26Z` / `b00526b153a31b36767991a4f5ce2cced435ee8e`. Builder — `golang:1.24.2-bookworm`, зависимости сверяются по upstream go.sum. Runtime-образы локальные `mtusi-lab4-minio:2025-04-22` и `mtusi-lab4-mc:2025-04-16`; скачивать готовый сервер преподавателя не требуется.
`bootstrap.sh` однократно создаёт пароль БД, root-credentials MinIO и отдельные API S3 credentials в `secrets/`; повтор их не меняет. Каталог 0700 на хосте, файлы доступны нужному uid через Compose secrets. Root S3 secrets получает только MinIO и minio-init; API/cleanup получают ограниченного пользователя. `.env`, secrets и volumes не входят в Git/build context. Учебные данные и credentials lab3 не копировались.
Безопасная остановка: `make down`, именованные volumes `postgres_data`, `redis_data`, `minio_data` сохраняются. Намеренное удаление всех учебных данных: `docker compose down --volumes`. Restart hook volumes не удаляет. `make test-fresh` удаляет только созданные им временные volumes отдельного проекта. После обычных тестов синтетические пользователи/сообщения/вложения остаются — delete не входит в обязательный API.
Отдельные команды:
```sh
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
# После отдельного публичного failover восстановить остановленную реплику:
docker compose up -d --no-recreate --wait --wait-timeout 90
python3 scripts/cluster.py wait
make cleanup-dry-run
# Явная безопасная очистка с обычным grace 300s:
docker compose run --rm --no-deps cleanup python -m messenger.cleanup --apply
```
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
клиент → Nginx → API ×2 ──→ PostgreSQL: пользователи/чаты/сообщения/сессии/metadata/связи
├─→ Redis: кэш публичного профиля
└─→ MinIO: оригинальные байты, приватный bucket
migrate → PostgreSQL minio-init → bucket/user/policy
cleanup → S3 listing + PostgreSQL recheck → удаление только старых неучтённых объектов
```
- **Перенос из lab3.** Сохранены Nginx, два экземпляра одного service api, общая PostgreSQL, Argon2id/Basic→Bearer, Redis cache-aside профиля, keyset-пагинация, X-Instance-Id и транзакционная идемпотентность сообщений. Сохранены публичные tests/contracts/README текущего skeleton. Служебные документы находятся отдельно в `lab4/docs/`.
- **Схема** (`003_attachments.sql`). `attachments`: UUID PK, chat_id/owner_id FK, составной FK на членство, filename, MIME, размер 1–10 MiB, SHA-256, уникальный object_key, created_at/updated_at. `message_attachments`: FK на сообщение/вложение, position 0–9, уникальны пара и позиция. Байты не хранятся в PostgreSQL. Проверка same-chat перед INSERT связи даёт 404 для чужого/неизвестного attachment_id. Любой участник может использовать доступное вложение своего чата.
- **Совместимость сообщения.** В ответах добавлен attachment_ids; в запросе по умолчанию []. Пустой/пробельный text допустим только с вложением. Список не содержит дублей, до 10 UUID, порядок сохраняется. Отсутствие списка и [] дают одинаковый fingerprint; пустой список не включается в fingerprint, поэтому сохраняется hash прежних text-only запросов lab3. Replay возвращает прежний Message 201, другой валидный body 409, доступ и сессия перепроверяются.
- **Пути и ACL.** Object key — `originals/<случайный UUID>`, не filename и не публичный URL. Имя — только отображаемое поле: 1–255 Unicode code points, не пустое, без slash/backslash/NUL/control. Сначала Bearer и членство, затем загрузка; metadata/content доступны только участникам, посторонний/неизвестный объект 404. Публичный JSON не раскрывает object key, bucket, endpoint или credentials.
- **Получение файла.** POST принимает сырой PNG/JPEG, не multipart. Content-Length проверяется заранее, но фактически полученные bytes тоже считаются. Тело читается через Request.stream в TemporaryFile на диске, SHA-256 считается по ходу. Лимит 10 MiB и абсолютное ожидание тела 30 с; слот загрузки ожидается до 1 с. Файл удаляется/закрывается в finally, постоянной локальной копии оригинала у API нет.
- **Проверка изображения.** Pillow определяет формат и размеры до load; MIME должен совпадать. >40 млн пикселей даёт 413 до полной распаковки. verify проверяет структуру, повторный open/load — декодирование. Повреждённый/неподдерживаемый файл 415, слишком большой 413, опасное имя 400. Текстовые PNG chunks ограничены 1 MiB. На API один decoder и не более 6 одновременно обрабатываемых upload: несколько больших распаковок не размножают расход RAM. Оригинальные байты не перекодируются; crop/thumbnail/очереди относятся к следующим работам.
- **Буферы Nginx.** На upload-route client_max_body_size 11 MiB, чтобы 10 MiB+1 дошёл до API и получил 413; client_body_buffer_size 64 KiB, остальное буферизуется на диск. `proxy_request_buffering on`, максимум 16 параллельных загрузок на LB, inactivity timeout тела 10 с, upload read/send timeout 30 с. Это сознательное дисковое буферизование, не обещание мгновенного приёма произвольно медленного клиента.
- **201 и согласованность.** После валидации API начинает PG-транзакцию и берёт advisory lock от object_key. Затем выполняются S3 PUT и INSERT metadata; 201 возвращается только после успешного COMMIT. У S3 и PostgreSQL нет общей атомарной транзакции. Подтверждённые metadata доступны всем API, оригинал находится в общем MinIO.
- **Компенсация неизвестного результата.** Ошибка PUT/SQL/COMMIT вызывает новую проверку под тем же lock. Объект удаляется только если БД точно подтверждает отсутствие metadata. При неизвестном COMMIT или недоступной БД слепого DeleteObject нет: уже учтённый файл должен остаться, неучтённый подберёт janitor. Обрыв неполного клиентского тела не начинает S3 PUT; аварийное завершение после PUT может оставить orphan.
- **Автоматическая очистка.** Отдельный cleanup-service каждые 60 с листает только originals/* страницами до 1000 объектов. Grace по умолчанию 300 с, минимум 60 с. Для старого кандидата pg_try_advisory_xact_lock и свежий SELECT повторно проверяют metadata; учтённый/свежий/активно заблокированный объект пропускается. При ошибке БД удаления не происходит. Dry-run не удаляет, повтор apply идемпотентен. Здесь реализована компенсация/orphan cleanup уровня 4, не полный исследовательский reconciler из трека 5 и не восстановление missing bytes.
- **Скачивание.** GET content возвращает StreamingBody S3 кусками 64 KiB через авторизованный API, без redirect. До 8 открытых S3-stream на API; close/release выполняется в finally ответа, включая ошибку отправки headers и disconnect. Content-Length сверяется с metadata. Variant в lab4 разрешён только original; прочие значения 400. Истинный MIME и RFC5987 filename передаются в headers.
- **MinIO и права.** minio-init идемпотентно создаёт bucket, запрещает anonymous, создаёт отдельного API user и policy. Get/Put/Delete разрешены только originals/* этого bucket; ListBucket — только для этого prefix; GetBucketLocation — для bucket. CreateBucket, чужие bucket/prefix и admin-права не выдаются. Console слушает 127.0.0.1:9001 внутри MinIO, не опубликована и недоступна из API-сети. Приватный bucket не заменяет ACL приложения.
- **Здоровье и таймауты.** Readiness требует PostgreSQL, совместимую схему и доступный S3 bucket с API credentials; Redis — необязательный кэш. Boto3 connect 1 с/read 3 с, одна попытка, пул/stream gate 8. PG-пул 8 на API, ожидание 2 с, statement 3 с/lock 2 с. S3 outage даёт 503 API либо 502/503/504 LB после исключения backend. Nginx OSS использует Docker DNS resolve и пассивные max_fails 1/fail_timeout 2 с; активный health_check Plus не используется. SIGTERM/rolling сохраняют схему lab3.
Роли, ресурсы и запуск:
| Service | Версия / ресурсы | Данные, сеть и готовность |
| --- | --- | --- |
| api×2 | `mtusi-lab4-api:lab4`; каждая 1 CPU/512 MiB, uid10001 | front+data, без host ports; HTTP ready, только ограниченные S3 secrets |
| lb | `nginx:1.28.0-alpine`; 0.5 CPU/128 MiB | только front; единственный public endpoint 127.0.0.1:8080 |
| postgres | `postgres:17.5-bookworm`; 1 CPU/512 MiB | internal data; postgres_data, pg_isready |
| redis | `redis:7.4.4-bookworm`; 0.25 CPU/128 MiB | internal data; redis_data, AOF, ping |
| minio | официальный release 2025-04-22, сборка из pinned source; 1 CPU/512 MiB, uid10002 | internal data; minio_data; HTTP /minio/health/ready; root secrets |
| minio-init | официальный mc release 2025-04-16, сборка; 0.25 CPU/128 MiB | root+API secrets, policy read-only, Exited(0) |
| migrate | образ API; 0.5 CPU/256 MiB | только DB secret, версии/lock/checksum, Exited(0) до API |
| cleanup | образ API; 0.25 CPU/192 MiB | DB+ограниченные S3 secrets, только data, цикл 60 с |
Python 3.13.5, библиотеки и транзитивные зависимости закреплены в requirements.lock. В .env реально используется SESSION_TTL_SECONDS; S3_ENDPOINT/REGION/BUCKET и *_FILE задаются Compose. Контрактные пределы 10 MiB/40 Mp не увеличиваются через переменные. Отдельные migrate и minio-init завершаются до API. Dockerfile приложения не запускает миграции каждой репликой.
## Доказательства критериев
Основные эксперименты — 2026-11-02; контрольный запуск — 2026-11-09. Среда: macOS/Apple M4 Pro, 24 GiB RAM; Docker VM 12 CPU и около 7.75 GiB. Ветка `solution/bpi2303-lukyanov`. Измерения относятся к текущему учебному стенду, пароль/токен в evidence не сохранялись.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Контракт и публичный API (3) | make check/validate-container/test | PASS: lab4 public API scenario, checksum, ACL и сообщения с ids | evidence/check.txt, openapi.txt, smoke.txt |
| Метаданные в PG, bytes в приватном S3 (3) | схема 003, security test, download | готовые originals в bucket, API не раскрывает key; anonymous 403 | migrations/003_attachments.sql, evidence/s3-security.json |
| PNG/JPEG и исходные bytes (3) | собственные тесты + smoke | 201 ready, SHA-256/размер/bytes совпадают, image-only сообщение валидно | evidence/student-tests.txt, smoke.txt |
| Byte/pixel/MIME/filename ограничения (3) | тесты повреждений, >10 MiB, chunked, >40 Mp header | 400/413/415; pixel limit до load, пользовательская ошибка не 500 | evidence/student-tests.txt |
| Upload на одной, download на другой (3) | verify_attachments.py failover | остановлен точно upload-instance; другой вернул тот же hash/bytes | evidence/attachment-failover.json |
| Полный restart с S3 volume (3) | verify_cluster.py persistence | прежняя сессия, metadata и оригинал доступны; реальные процессы перезапущены | evidence/persistence.txt, persistence.json |
| Прежняя балансировка/идемпотентность (3) | test-cluster, собственные регрессии | concurrent replay, failover/SIGTERM/rolling проходят | evidence/concurrent-idempotency.json, public-failover.txt, sequential-failover.json, graceful-shutdown.json, rolling-update.json |
| Все отрицательные случаи файлов/чужих ids (4) | make test-student | 21 тест PASS, включая MIME, corrupt/oversize, чужой attachment и другой чат | evidence/student-tests.txt |
| Компенсация после PUT/DB failure (4) | SQL trigger перед metadata INSERT, затем реальный SIGKILL | SQL-сбой 503 компенсирован; orphan после kill обнаружен и удалён после grace | evidence/student-tests.txt, orphan-cleanup.json |
| Безопасная очистка (4) | dry-run/apply/repeat; fresh/tracked/locked | dry-run: 0 удалений; apply удалил orphan; повтор 0; свежий/используемый/locked сохранены | evidence/orphan-cleanup.json |
| Потоковая обработка и RAM при ≥10 загрузках (4) | make uploads | 10/10 одновременно, 2 API, 0 ошибок/OOM; пик 205.824/214.730 MiB при 512 MiB | evidence/concurrent-uploads.json |
| Bucket bootstrap идемпотентен, IAM ограничен (4) | повтор minio-init; прямые S3-запросы | objects/metadata не изменились; anonymous 403, CreateBucket 403, другой prefix: 403 | evidence/s3-security.json |
| Консоль не открыта (4) | socket из API к minio:9001; infrastructure snapshot | соединение не установлено, root secret в API не смонтирован, host ports только у LB | evidence/s3-security.json, infrastructure.json |
| Миграции/инициализация с нуля | make test-fresh | schema 001–003 и private bucket на новых volumes, тесты PASS | evidence/fresh-migrations.txt |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Pytest также выводит предупреждение старой закреплённой botocore об utcnow в Python 3.13 и диагностическое предупреждение Pillow в тесте pixel-bomb; проверки результата 413 и все assertions проходят. Публичные проверки и контракт не ослаблялись.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Исследовательский трек 5 не выполнялся. Файлы синтетические. Обрыв тела и сбой между PUT/COMMIT проверяются раздельно: одного успешного API-запроса недостаточно для доказательства компенсации.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потери |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Upload → stop обслужившей API → download | файл читается через другую API | тот же SHA-256 и bytes, другой alias | 1.988 с от старта действия / потерь нет |
| Обрыв HTTP upload | неполное тело не создаёт объект/metadata | из 500000 объявленных передано 108 bytes, TCP закрыт; counts не изменились | orphan нет |
| Ошибка INSERT metadata после S3 PUT | rollback PG и компенсация S3 | HTTP 503, нет строки и нового объекта | подтверждения не было |
| SIGKILL после PUT, до COMMIT | неподтверждённый объект может остаться и будет очищен | наблюдался exit 137; объект есть, metadata нет; после 61 с реального ожидания dry-run показал кандидата, apply удалил его | orphan устранён, 201 не выдан |
| Repeat cleanup, fresh/linked/locked | безопасные повторы | повтор не удаляет объекты; свежий и используемый сохранены, объект под lock сохранён до unlock | потерь используемых данных нет |
| MinIO недоступен | ограниченная ошибка без ложного 201 | upload 503 за 0.2487 с; после старта прежний оригинал читается | тот же токен, bytes сохранены |
| Полный restart API/PG/Redis/MinIO/cleanup/Nginx | сессия и оригинал переживают restart | public persistence PASS | именованные volumes сохранены |
**Методика crash/cleanup:** временный BEFORE INSERT trigger задерживает ровно одно имя файла через pg_sleep(2). После наблюдения INSERT в pg_stat_activity и появления объекта в S3 API получает SIGKILL. Metadata не зафиксирована, 201 не отправлен. Контейнер восстановлен, trigger удалён. После настоящего 61-секундного ожидания применяется grace 60 с. Дополнительно проверены старый объект под advisory lock, свежий объект и старое вложение, связанное с сообщением. После снятия lock его тестовый orphan также удалён. В обычном запуске grace 300 с, цикл 60 с; при недоступной БД уборка откладывается.
**Память при десяти загрузках:** воспроизводимый RGB PNG 3000×1000, 9002318 bytes, zlib level 0, строки из SHAKE-256. Один неизменяемый payload используется десятью клиентскими потоками с barrier. Все десять HTTP-интервалов перекрывались 0.345 с, завершились 201, обе API наблюдались, ошибок 0. Время группы 1.563 с. Один оригинал скачан другим участником и побайтно сверен.
Перед опытом API пересозданы, чтобы memory.peak и VmHWM относились к этому запуску. Peak включает startup/bootstrap/загрузки/контрольное скачивание. Kernel high-water mark не пропускает короткие пики между samples:
| API-процесс | Cgroup memory peak | PID 1 VmHWM | Лимит | OOM |
| --- | --- | --- | --- | --- |
| Первый | 205.824 MiB | 182.871 MiB | 512 MiB | 0 |
| Второй | 214.730 MiB | 183.293 MiB | 512 MiB | 0 |
Cgroup включает cache и вспомогательные процессы, VmHWM — resident peak API. Профиль, версии, limits, baseline/final, интервалы запросов и checksum — в evidence/concurrent-uploads.json. Опыт показывает расход памяти этого профиля, а не максимальный throughput или универсальный RSS для всех допустимых JPEG/PNG. Сценарии lab3 под текстовой нагрузкой сохранены отдельно; их числа не заменяют эксперимент с файлами.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Один Docker-host, одна PostgreSQL и один MinIO: отказ узла/диска и HA хранилищ не покрываются. Named volume не заменяет backup. Потерянные bytes из missing object janitor восстановить не может.
- S3 и PostgreSQL неатомарны. Между PUT и COMMIT есть окно orphan; компенсация и grace-cleanup закрывают его после восстановления зависимостей. При недоступной БД удаление запрещено; время уборки не является SLA.
- Janitor удаляет только неучтённые старые объекты. Учтённое, но никуда не прикреплённое вложение сохраняется: в обязательном API нет delete/retention для таких файлов.
- Readiness считает S3 критичной зависимостью. При S3 outage Nginx может исключить обе API, и text-only запросы тоже станут недоступны, хотя PG жив. Внутренний live процесса остаётся 200.
- Таймауты SDK/Nginx относятся к отдельным операциям I/O. После начала HTTP download нельзя заменить частичное 200 структурированным 503; клиент проверяет размер/checksum.
- Нагрузочный замер: десять одновременных загрузок по 9 MB, две API, один прогон. Исследования максимального RPS, presigned upload, thumbnail/crop и фоновой обработки lab5 не проводились.
- MinIO собран из закреплённого исторического релиза; стенд локальный и учебный. HTTP/Basic только на localhost с синтетическими данными; TLS, rotation, secret manager и полное разделение DB privileges оставлены следующим работам.
- Между страницами нет общего snapshot. Сохраняются оговорённые в lab3 ограничения часов и конкурентных вставок. Breaking DB migrations потребуют отдельной стратегии.
+150
View File
@@ -0,0 +1,150 @@
name: mtusi-lab4
services:
postgres:
image: postgres:17.5-bookworm
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql/data]
networks: [data]
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 30
mem_limit: 512m
cpus: 1.0
redis:
image: redis:7.4.4-bookworm
command: [redis-server, --appendonly, 'yes', --maxmemory, 64mb, --maxmemory-policy, allkeys-lru]
volumes: [redis_data:/data]
networks: [data]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 128m
cpus: 0.25
migrate:
image: mtusi-lab4-api:lab4
command: [python, -m, messenger.migrate]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
networks: [data]
depends_on:
postgres: {condition: service_healthy}
restart: 'no'
mem_limit: 256m
cpus: 0.5
api:
image: mtusi-lab4-api:lab4
build: .
scale: 2
environment:
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
POOL_MAX_SIZE: '8'
S3_ENDPOINT: http://minio:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
networks: [front, data]
depends_on:
migrate: {condition: service_completed_successfully}
redis: {condition: service_healthy}
minio-init: {condition: service_completed_successfully}
stop_grace_period: 10s
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"]
interval: 2s
timeout: 5s
retries: 30
mem_limit: 512m
cpus: 1.0
minio:
image: mtusi-lab4-minio:2025-04-22
build:
context: .
dockerfile: infra/minio/Dockerfile
target: minio
secrets: [minio_root_user, minio_root_password]
volumes: [minio_data:/data]
networks: [data]
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:9000/minio/health/ready',timeout=2)"]
interval: 2s
timeout: 3s
retries: 30
stop_grace_period: 10s
mem_limit: 512m
cpus: 1.0
minio-init:
image: mtusi-lab4-mc:2025-04-16
build:
context: .
dockerfile: infra/minio/Dockerfile
target: mc
secrets: [minio_root_user, minio_root_password, s3_access_key_id, s3_secret_access_key]
volumes:
- ./infra/minio/bootstrap.sh:/config/bootstrap.sh:ro
- ./infra/minio/api-policy.json:/config/api-policy.json:ro
networks: [data]
depends_on:
minio: {condition: service_healthy}
restart: 'no'
mem_limit: 128m
cpus: 0.25
cleanup:
image: mtusi-lab4-api:lab4
command: [python, -m, messenger.cleanup, --loop, --apply, --grace-seconds, '300', --interval, '60']
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: http://minio:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
networks: [data]
depends_on:
migrate: {condition: service_completed_successfully}
minio-init: {condition: service_completed_successfully}
mem_limit: 192m
cpus: 0.25
lb:
image: nginx:1.28.0-alpine
volumes: ['./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro']
networks: [front]
ports: ['127.0.0.1:8080:8080']
depends_on:
api: {condition: service_healthy}
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
interval: 1s
timeout: 8s
retries: 30
mem_limit: 128m
cpus: 0.5
networks:
front:
data:
internal: true
secrets:
postgres_password: {file: ./secrets/postgres_password}
database_url: {file: ./secrets/database_url}
cache_url: {file: ./secrets/cache_url}
minio_root_user: {file: ./secrets/minio_root_user}
minio_root_password: {file: ./secrets/minio_root_password}
s3_access_key_id: {file: ./secrets/s3_access_key_id}
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
volumes:
postgres_data:
redis_data:
minio_data:
+18
View File
@@ -0,0 +1,18 @@
{
"result": "PASS",
"stopped": {
"container": "01ac5da3549e234a658a6202029fec3661951caf9cd6230f71536579465b51df",
"name": "mtusi-lab4-api-1",
"instance": "68057e4f671d4bc4beebe6446a39f551",
"release": "lab4-v1",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"stopped_at_utc": "2026-11-02T09:10:52.334643+00:00",
"stop_seconds": 0.415,
"exit_code": 0,
"left_stopped": true
},
"download_instance": "4bd9e299c91d4af99e1b94bab75f8f3d",
"same_bytes": true,
"same_sha256": true,
"seconds_from_action_start": 1.988
}
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+11
View File
@@ -0,0 +1,11 @@
{
"result": "PASS",
"parallel_requests": 8,
"observed_instances": [
"3ca208a978e441ccba13625143527d66",
"de89643f69534e43bc1111510e0a993e"
],
"responses_201": 8,
"messages_created": 1,
"different_body_status": 409
}
+612
View File
@@ -0,0 +1,612 @@
{
"captured_at_utc": "2026-11-02T09:15:37.434138+00:00",
"result": "PASS",
"context": {
"git_commit": "ec540a1a725862a8619fcb758daec4ad197f0dd5",
"tracked_worktree_dirty": true,
"host_python": "3.13.7",
"host_os": "macOS-27.2-arm64-arm-64bit-Mach-O",
"docker_version": "29.8.2",
"compose_version": "5.6.0",
"contract_version": "1.4.0",
"endpoint": {
"scheme": "http",
"host": "localhost",
"port": 8080
}
},
"input": {
"format": "PNG",
"mode": "RGB",
"width": 3000,
"height": 1000,
"size_bytes": 9002318,
"sha256": "9bb84f9e461995a5e031a0678da59c6dea8d963a157e590a9798d9030437c3fe",
"generator": "SHAKE-256 lab4-concurrent-upload-v1 row seeds; filter 0; zlib level 0",
"same_immutable_bytes_for_all_requests": true
},
"concurrency": 10,
"reset": {
"command": "docker compose up -d --no-deps --force-recreate api",
"purpose": "Reset API cgroup memory.peak and PID 1 VmHWM; retain all volumes"
},
"peak_scope": "Fresh API startup, bootstrap, ten uploads and representative download; no peak sampling gaps",
"cluster_before": [
{
"service": "api",
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
"name": "mtusi-lab4-api-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:15:38.382323712Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data",
"mtusi-lab4_front"
],
"http_status": 200,
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
"release": "lab4-v1"
},
{
"service": "api",
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
"name": "mtusi-lab4-api-2",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:15:38.308471337Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data",
"mtusi-lab4_front"
],
"http_status": 200,
"instance": "de89643f69534e43bc1111510e0a993e",
"release": "lab4-v1"
},
{
"service": "cleanup",
"container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6",
"name": "mtusi-lab4-cleanup-1",
"status": "running",
"health": null,
"started_at": "2026-11-02T09:10:38.357370005Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 201326592,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "lb",
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
"name": "mtusi-lab4-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:03:39.330794254Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab4_front"
]
},
{
"service": "migrate",
"container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2",
"name": "mtusi-lab4-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-02T09:10:37.212308463Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "minio",
"container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36",
"name": "mtusi-lab4-minio-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:12:16.586994384Z",
"image": "mtusi-lab4-minio:2025-04-22",
"image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_minio_data"
],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "minio-init",
"container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641",
"name": "mtusi-lab4-minio-init-1",
"status": "exited",
"health": null,
"started_at": "2026-11-02T09:10:37.288107921Z",
"image": "mtusi-lab4-mc:2025-04-16",
"image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "postgres",
"container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2",
"name": "mtusi-lab4-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:03:32.508777334Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_postgres_data"
],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "redis",
"container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a",
"name": "mtusi-lab4-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:03:32.445843167Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_redis_data"
],
"networks": [
"mtusi-lab4_data"
]
}
],
"baseline": {
"3bd3bbc5d37c4b5a97dc188076f363dd": {
"source": "cgroup-v2",
"cgroup_peak_bytes": 96063488,
"cgroup_current_bytes": 87937024,
"pid1_vmhwm_bytes": 96518144,
"pid1_vmrss_bytes": 96518144,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0,
"sock_throttled": 0
}
},
"de89643f69534e43bc1111510e0a993e": {
"source": "cgroup-v2",
"cgroup_peak_bytes": 104452096,
"cgroup_current_bytes": 87433216,
"pid1_vmhwm_bytes": 96370688,
"pid1_vmrss_bytes": 96370688,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0,
"sock_throttled": 0
}
}
},
"memory_limits": [
{
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
{
"instance": "de89643f69534e43bc1111510e0a993e",
"memory_bytes": 536870912,
"nano_cpus": 1000000000
}
],
"requests": [
{
"index": 0,
"start_seconds": 0.9074667920067441,
"status": 201,
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
"attachment_id": "14a5f6e5-0973-4b7f-98e2-81a9b7ba6ccb",
"metadata_verified": true,
"end_seconds": 1.4336796249990584
},
{
"index": 1,
"start_seconds": 0.9073489169968525,
"status": 201,
"instance": "de89643f69534e43bc1111510e0a993e",
"attachment_id": "79f74d1b-624f-497e-b80d-e2605dc22427",
"metadata_verified": true,
"end_seconds": 1.5235847080039093
},
{
"index": 2,
"start_seconds": 0.9075777499965625,
"status": 201,
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
"attachment_id": "c5e24f93-8f8b-49fc-9e71-13161f08be4b",
"metadata_verified": true,
"end_seconds": 1.2526268750079907
},
{
"index": 3,
"start_seconds": 0.907239958003629,
"status": 201,
"instance": "de89643f69534e43bc1111510e0a993e",
"attachment_id": "480d346b-f983-45a3-9a78-05064f27ef06",
"metadata_verified": true,
"end_seconds": 1.352467999997316
},
{
"index": 4,
"start_seconds": 0.9076472919987282,
"status": 201,
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
"attachment_id": "9e96aac9-21a4-4b27-b600-881e440e17b7",
"metadata_verified": true,
"end_seconds": 1.5511514579993673
},
{
"index": 5,
"start_seconds": 0.9077015420043608,
"status": 201,
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
"attachment_id": "85dde29d-dccf-43a8-b634-4fb380ce555f",
"metadata_verified": true,
"end_seconds": 1.5152983750012936
},
{
"index": 6,
"start_seconds": 0.9077392080071149,
"status": 201,
"instance": "de89643f69534e43bc1111510e0a993e",
"attachment_id": "63ab944f-fd09-465e-bd9a-ad033cf79477",
"metadata_verified": true,
"end_seconds": 1.5623937500058673
},
{
"index": 7,
"start_seconds": 0.9075430829980178,
"status": 201,
"instance": "de89643f69534e43bc1111510e0a993e",
"attachment_id": "4b5e0c15-4a6c-47a4-9741-74cd6ac3b3db",
"metadata_verified": true,
"end_seconds": 1.5540448750107316
},
{
"index": 8,
"start_seconds": 0.9074074170057429,
"status": 201,
"instance": "de89643f69534e43bc1111510e0a993e",
"attachment_id": "38272fe7-3a6a-4064-9664-f7346a36431a",
"metadata_verified": true,
"end_seconds": 1.5352119580056751
},
{
"index": 9,
"start_seconds": 0.9075136250030482,
"status": 201,
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
"attachment_id": "5c124808-e996-4e34-bfe6-16f20fbe6895",
"metadata_verified": true,
"end_seconds": 1.5333833750046324
}
],
"elapsed_seconds": 1.562777917002677,
"peak_active_requests": 10,
"all_ten_overlap_seconds": 0.3448876670008758,
"all_ten_overlapping": true,
"successful_count": 10,
"error_count": 0,
"observed_instances": [
"3bd3bbc5d37c4b5a97dc188076f363dd",
"de89643f69534e43bc1111510e0a993e"
],
"representative_download": {
"verified_as": "bob (chat member)",
"matches_original_bytes": true,
"size_bytes": 9002318,
"sha256": "9bb84f9e461995a5e031a0678da59c6dea8d963a157e590a9798d9030437c3fe"
},
"final": {
"3bd3bbc5d37c4b5a97dc188076f363dd": {
"source": "cgroup-v2",
"cgroup_peak_bytes": 215822336,
"cgroup_current_bytes": 183332864,
"pid1_vmhwm_bytes": 191754240,
"pid1_vmrss_bytes": 191754240,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0,
"sock_throttled": 0
}
},
"de89643f69534e43bc1111510e0a993e": {
"source": "cgroup-v2",
"cgroup_peak_bytes": 225161216,
"cgroup_current_bytes": 182919168,
"pid1_vmhwm_bytes": 192196608,
"pid1_vmrss_bytes": 192196608,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0,
"sock_throttled": 0
}
}
},
"cluster_after": [
{
"service": "api",
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
"name": "mtusi-lab4-api-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:15:38.382323712Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data",
"mtusi-lab4_front"
],
"http_status": 200,
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
"release": "lab4-v1"
},
{
"service": "api",
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
"name": "mtusi-lab4-api-2",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:15:38.308471337Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data",
"mtusi-lab4_front"
],
"http_status": 200,
"instance": "de89643f69534e43bc1111510e0a993e",
"release": "lab4-v1"
},
{
"service": "cleanup",
"container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6",
"name": "mtusi-lab4-cleanup-1",
"status": "running",
"health": null,
"started_at": "2026-11-02T09:10:38.357370005Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 201326592,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "lb",
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
"name": "mtusi-lab4-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:03:39.330794254Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab4_front"
]
},
{
"service": "migrate",
"container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2",
"name": "mtusi-lab4-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-02T09:10:37.212308463Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "minio",
"container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36",
"name": "mtusi-lab4-minio-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:12:16.586994384Z",
"image": "mtusi-lab4-minio:2025-04-22",
"image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_minio_data"
],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "minio-init",
"container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641",
"name": "mtusi-lab4-minio-init-1",
"status": "exited",
"health": null,
"started_at": "2026-11-02T09:10:37.288107921Z",
"image": "mtusi-lab4-mc:2025-04-16",
"image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "postgres",
"container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2",
"name": "mtusi-lab4-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:03:32.508777334Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_postgres_data"
],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "redis",
"container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a",
"name": "mtusi-lab4-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:03:32.445843167Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_redis_data"
],
"networks": [
"mtusi-lab4_data"
]
}
],
"oom_killed": {
"mtusi-lab4-api-1": false,
"mtusi-lab4-api-2": false
},
"per_node_peaks": {
"3bd3bbc5d37c4b5a97dc188076f363dd": {
"source": "cgroup-v2",
"cgroup_peak_bytes": 215822336,
"cgroup_current_bytes": 183332864,
"pid1_vmhwm_bytes": 191754240,
"pid1_vmrss_bytes": 191754240,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0,
"sock_throttled": 0
}
},
"de89643f69534e43bc1111510e0a993e": {
"source": "cgroup-v2",
"cgroup_peak_bytes": 225161216,
"cgroup_current_bytes": 182919168,
"pid1_vmhwm_bytes": 192196608,
"pid1_vmrss_bytes": 192196608,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0,
"sock_throttled": 0
}
}
}
}
+20
View File
@@ -0,0 +1,20 @@
{
"result": "PASS",
"redis": {
"profile_status": 200,
"fallback_seconds": 0.0132,
"anonymous": 401,
"readiness": 200
},
"postgres": {
"authenticated_status": 503,
"response_seconds": 0.0047,
"anonymous_status": 401,
"internal_api_liveness": [
200,
200
],
"note": "Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable."
},
"same_session_after_recovery": true
}
+20
View File
@@ -0,0 +1,20 @@
./scripts/check_fresh_migrations.sh
Applied 001_initial.sql
Applied 002_idempotency.sql
Applied 003_attachments.sql
Migrations up to date
Migrations up to date
..................... [100%]
=============================== warnings summary ===============================
student-tests/test_attachments.py: 13 warnings
student-tests/test_idempotency.py: 1 warning
/usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
datetime_now = datetime.datetime.utcnow()
student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename
/usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack.
warnings.warn(
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
21 passed, 15 warnings in 4.23s
PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.
+16
View File
@@ -0,0 +1,16 @@
{
"result": "PASS",
"target_container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
"target_instance": "69d3312a6f6b4f7bb50651c8632e196a",
"fixture": "temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards",
"in_flight_observed_in_pg_stat_activity": true,
"in_flight_response": 201,
"new_tcp_after_sigterm": {
"accepted": false,
"error_type": "ConnectionRefusedError"
},
"target_still_running_when_new_tcp_rejected": true,
"exit_code": 0,
"shutdown_seconds_from_sigterm": 2.0927,
"confirmed_message_preserved": true
}
+179
View File
@@ -0,0 +1,179 @@
{
"result": "PASS",
"containers": [
{
"service": "api",
"container": "102a32862f73ac850881b369827c3d0fef35d41445893bd6807200759f201e3b",
"name": "mtusi-lab4-api-3",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:26:04.73288638Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:e53ef1cc7aef252b12b8f7e0136c8836a5ab075152b194a609e6c6fa1f4de351",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data",
"mtusi-lab4_front"
],
"http_status": 200,
"instance": "3715a1c3cd074e9f952ad3b9bb646d54",
"release": "lab4-ce567129"
},
{
"service": "api",
"container": "2089d486fe8dbdf85cfda30cfe2ea79de996f070343cb076dc3bceb83d18e8a0",
"name": "mtusi-lab4-api-4",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:26:10.354473716Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:e53ef1cc7aef252b12b8f7e0136c8836a5ab075152b194a609e6c6fa1f4de351",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data",
"mtusi-lab4_front"
],
"http_status": 200,
"instance": "0f0ee5b42e754b7f9aa359e746294e8f",
"release": "lab4-ce567129"
},
{
"service": "cleanup",
"container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6",
"name": "mtusi-lab4-cleanup-1",
"status": "running",
"health": null,
"started_at": "2026-11-02T09:24:53.220476166Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 201326592,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "lb",
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
"name": "mtusi-lab4-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:24:50.272136138Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab4_front"
]
},
{
"service": "migrate",
"container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2",
"name": "mtusi-lab4-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-02T09:25:13.130777342Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "minio",
"container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36",
"name": "mtusi-lab4-minio-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:24:50.261138388Z",
"image": "mtusi-lab4-minio:2025-04-22",
"image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_minio_data"
],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "minio-init",
"container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641",
"name": "mtusi-lab4-minio-init-1",
"status": "exited",
"health": null,
"started_at": "2026-11-02T09:25:13.207299134Z",
"image": "mtusi-lab4-mc:2025-04-16",
"image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "postgres",
"container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2",
"name": "mtusi-lab4-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:26:21.713520555Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_postgres_data"
],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "redis",
"container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a",
"name": "mtusi-lab4-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:26:20.609059638Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_redis_data"
],
"networks": [
"mtusi-lab4_data"
]
}
]
}
+7
View File
@@ -0,0 +1,7 @@
{
"result": "PASS",
"declared_bytes": 500000,
"sent_bytes": 108,
"metadata_unchanged": true,
"s3_object_count_unchanged": true
}
+11
View File
@@ -0,0 +1,11 @@
{
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
"name": "mtusi-lab4-api-2",
"instance": "6e358c3478904b0e97230b8ceeb17c24",
"release": "lab4-v1",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"stopped_at_utc": "2026-11-02T09:25:37.448855+00:00",
"stop_seconds": 0.5935,
"exit_code": 0,
"left_stopped": true
}
+11
View File
@@ -0,0 +1,11 @@
{
"result": "PASS",
"version": "nginx version: nginx/1.28.0",
"image": "nginx:1.28.0-alpine",
"public_server_header": "nginx",
"config_syntax": "PASS",
"observed_instances": [
"3ca208a978e441ccba13625143527d66",
"de89643f69534e43bc1111510e0a993e"
]
}
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/cluster.py exec python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+52
View File
@@ -0,0 +1,52 @@
{
"result": "PASS",
"fault": "SIGKILL API after successful S3PUT, before metadata COMMIT",
"crash_exit_code": 137,
"target_container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
"upload_acknowledged": false,
"orphan_confirmed": true,
"grace_wait_seconds": 61,
"dry_run": {
"mode": "dry-run",
"grace_seconds": 60,
"listed": 29,
"fresh": 1,
"tracked": 26,
"locked": 1,
"orphans": 1,
"deleted": 0
},
"apply": {
"mode": "apply",
"grace_seconds": 60,
"listed": 29,
"fresh": 1,
"tracked": 26,
"locked": 1,
"orphans": 1,
"deleted": 1
},
"repeat": {
"mode": "apply",
"grace_seconds": 60,
"listed": 28,
"fresh": 1,
"tracked": 26,
"locked": 1,
"orphans": 0,
"deleted": 0
},
"after_lock_release": {
"mode": "apply",
"grace_seconds": 60,
"listed": 28,
"fresh": 1,
"tracked": 26,
"locked": 0,
"orphans": 1,
"deleted": 1
},
"fresh_preserved": true,
"tracked_and_linked_preserved": true,
"active_lock_preserved": true
}
+356
View File
@@ -0,0 +1,356 @@
{
"result": "PASS",
"before": [
{
"service": "api",
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
"name": "mtusi-lab4-api-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:23:34.501426213Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data",
"mtusi-lab4_front"
],
"http_status": 200,
"instance": "3ca208a978e441ccba13625143527d66",
"release": "lab4-v1"
},
{
"service": "api",
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
"name": "mtusi-lab4-api-2",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:15:38.308471337Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data",
"mtusi-lab4_front"
],
"http_status": 200,
"instance": "de89643f69534e43bc1111510e0a993e",
"release": "lab4-v1"
},
{
"service": "cleanup",
"container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6",
"name": "mtusi-lab4-cleanup-1",
"status": "running",
"health": null,
"started_at": "2026-11-02T09:10:38.357370005Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 201326592,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "lb",
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
"name": "mtusi-lab4-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:03:39.330794254Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab4_front"
]
},
{
"service": "migrate",
"container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2",
"name": "mtusi-lab4-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-02T09:10:37.212308463Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "minio",
"container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36",
"name": "mtusi-lab4-minio-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:12:16.586994384Z",
"image": "mtusi-lab4-minio:2025-04-22",
"image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_minio_data"
],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "minio-init",
"container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641",
"name": "mtusi-lab4-minio-init-1",
"status": "exited",
"health": null,
"started_at": "2026-11-02T09:10:37.288107921Z",
"image": "mtusi-lab4-mc:2025-04-16",
"image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "postgres",
"container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2",
"name": "mtusi-lab4-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:03:32.508777334Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_postgres_data"
],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "redis",
"container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a",
"name": "mtusi-lab4-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:03:32.445843167Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_redis_data"
],
"networks": [
"mtusi-lab4_data"
]
}
],
"after": [
{
"service": "api",
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
"name": "mtusi-lab4-api-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:24:50.617456221Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data",
"mtusi-lab4_front"
],
"http_status": 200,
"instance": "c5dd801e4df142e599da4bafc592e9d7",
"release": "lab4-v1"
},
{
"service": "api",
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
"name": "mtusi-lab4-api-2",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:24:50.685437846Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data",
"mtusi-lab4_front"
],
"http_status": 200,
"instance": "6e358c3478904b0e97230b8ceeb17c24",
"release": "lab4-v1"
},
{
"service": "cleanup",
"container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6",
"name": "mtusi-lab4-cleanup-1",
"status": "running",
"health": null,
"started_at": "2026-11-02T09:24:53.220476166Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 201326592,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "lb",
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
"name": "mtusi-lab4-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:24:50.272136138Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab4_front"
]
},
{
"service": "migrate",
"container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2",
"name": "mtusi-lab4-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-02T09:24:53.965337083Z",
"image": "mtusi-lab4-api:lab4",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "minio",
"container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36",
"name": "mtusi-lab4-minio-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:24:50.261138388Z",
"image": "mtusi-lab4-minio:2025-04-22",
"image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_minio_data"
],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "minio-init",
"container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641",
"name": "mtusi-lab4-minio-init-1",
"status": "exited",
"health": null,
"started_at": "2026-11-02T09:24:53.907159458Z",
"image": "mtusi-lab4-mc:2025-04-16",
"image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "postgres",
"container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2",
"name": "mtusi-lab4-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:24:50.284073804Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_postgres_data"
],
"networks": [
"mtusi-lab4_data"
]
},
{
"service": "redis",
"container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a",
"name": "mtusi-lab4-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:24:50.227454096Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab4_redis_data"
],
"networks": [
"mtusi-lab4_data"
]
}
],
"public_test": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence."
}
+2
View File
@@ -0,0 +1,2 @@
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+15
View File
@@ -0,0 +1,15 @@
{
"result": "PASS",
"public_test": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 134, \"failed_attempts\": 0, \"first_recovery_seconds\": 1.104}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.",
"stopped": {
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
"name": "mtusi-lab4-api-1",
"instance": "c5dd801e4df142e599da4bafc592e9d7",
"release": "lab4-v1",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"stopped_at_utc": "2026-11-02T09:24:58.384960+00:00",
"stop_seconds": 0.4473,
"exit_code": 0,
"left_stopped": true
}
}
+3
View File
@@ -0,0 +1,3 @@
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 134, "failed_attempts": 0, "first_recovery_seconds": 1.104}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+171
View File
@@ -0,0 +1,171 @@
{
"result": "PASS",
"before": [
{
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
"name": "mtusi-lab4-api-1",
"instance": "0e4e9b12996647ac8d03d03d79ae64fe",
"release": "lab4-v1",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961"
},
{
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
"name": "mtusi-lab4-api-2",
"instance": "427b43f714ff4bc3b99698dc90cafa3b",
"release": "lab4-v1",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961"
}
],
"after": [
{
"container": "102a32862f73ac850881b369827c3d0fef35d41445893bd6807200759f201e3b",
"name": "mtusi-lab4-api-3",
"instance": "3715a1c3cd074e9f952ad3b9bb646d54",
"release": "lab4-ce567129",
"image_id": "sha256:e53ef1cc7aef252b12b8f7e0136c8836a5ab075152b194a609e6c6fa1f4de351"
},
{
"container": "2089d486fe8dbdf85cfda30cfe2ea79de996f070343cb076dc3bceb83d18e8a0",
"name": "mtusi-lab4-api-4",
"instance": "0f0ee5b42e754b7f9aa359e746294e8f",
"release": "lab4-ce567129",
"image_id": "sha256:e53ef1cc7aef252b12b8f7e0136c8836a5ab075152b194a609e6c6fa1f4de351"
}
],
"target_release": "lab4-ce567129",
"lb_before": {
"service": "lb",
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
"name": "mtusi-lab4-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:24:50.272136138Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab4_front"
]
},
"lb_after": {
"service": "lb",
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
"name": "mtusi-lab4-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T09:24:50.272136138Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab4_front"
]
},
"lb_not_restarted": true,
"build_seconds": 2.7115,
"migration_output": "Migrations up to date",
"replacements": [
{
"old_container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
"new_container": "102a32862f73ac850881b369827c3d0fef35d41445893bd6807200759f201e3b",
"old_ip": "172.25.0.2",
"new_ip": "172.25.0.5",
"dns_discovered_new_address": true,
"instances_during_start_first": [
"0e4e9b12996647ac8d03d03d79ae64fe",
"3715a1c3cd074e9f952ad3b9bb646d54",
"427b43f714ff4bc3b99698dc90cafa3b"
],
"observed_instances": [
"3715a1c3cd074e9f952ad3b9bb646d54",
"427b43f714ff4bc3b99698dc90cafa3b"
],
"old_exit_code": 0,
"seconds": 5.6289
},
{
"old_container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
"new_container": "2089d486fe8dbdf85cfda30cfe2ea79de996f070343cb076dc3bceb83d18e8a0",
"old_ip": "172.25.0.3",
"new_ip": "172.25.0.2",
"dns_discovered_new_address": true,
"instances_during_start_first": [
"0f0ee5b42e754b7f9aa359e746294e8f",
"3715a1c3cd074e9f952ad3b9bb646d54",
"427b43f714ff4bc3b99698dc90cafa3b"
],
"observed_instances": [
"0f0ee5b42e754b7f9aa359e746294e8f",
"3715a1c3cd074e9f952ad3b9bb646d54"
],
"old_exit_code": 0,
"seconds": 5.4987
}
],
"during_replacement": {
"seconds": 11.1277,
"concurrency": 4,
"successful_requests": 8499,
"failed_attempts": 0,
"successful_rps": 763.77,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 4.34,
"p95": 7.505,
"p99": 37.719
},
"instances": [
"0e4e9b12996647ac8d03d03d79ae64fe",
"0f0ee5b42e754b7f9aa359e746294e8f",
"3715a1c3cd074e9f952ad3b9bb646d54",
"427b43f714ff4bc3b99698dc90cafa3b"
]
},
"total": {
"seconds": 20.7958,
"concurrency": 4,
"successful_requests": 15875,
"failed_attempts": 0,
"successful_rps": 763.377,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 4.479,
"p95": 7.406,
"p99": 28.635
},
"instances": [
"0e4e9b12996647ac8d03d03d79ae64fe",
"0f0ee5b42e754b7f9aa359e746294e8f",
"3715a1c3cd074e9f952ad3b9bb646d54",
"427b43f714ff4bc3b99698dc90cafa3b"
]
},
"data": {
"confirmed_unique_messages": 15875,
"history_messages": 15875,
"confirmed_preserved": true,
"duplicate_ids": 0,
"duplicate_texts": 0
},
"change": "New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate."
}
+7
View File
@@ -0,0 +1,7 @@
{
"result": "PASS",
"upload_status_during_outage": 503,
"response_seconds": 0.2487,
"original_preserved_after_restart": true,
"same_session_after_recovery": true
}
+10
View File
@@ -0,0 +1,10 @@
{
"result": "PASS",
"anonymous_get": 403,
"create_bucket_with_api_credentials": 403,
"list_other_prefix": 403,
"console_reachable_from_api": false,
"root_secret_mounted_in_api": false,
"bootstrap_repeated_without_data_loss": true,
"bootstrap_output": "PASS: private bucket and restricted API policy ready (credentials hidden)."
}
+135
View File
@@ -0,0 +1,135 @@
{
"result": "PASS",
"baseline": {
"seconds": 3.0101,
"concurrency": 4,
"successful_requests": 2029,
"failed_attempts": 0,
"successful_rps": 674.073,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 4.838,
"p95": 7.446,
"p99": 24.831
},
"instances": [
"118d370a438c4c14aabe117988bc86e9",
"6e358c3478904b0e97230b8ceeb17c24"
]
},
"phases": [
{
"stopped": {
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
"name": "mtusi-lab4-api-1",
"instance": "118d370a438c4c14aabe117988bc86e9",
"release": "lab4-v1",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"stopped_at_utc": "2026-11-02T09:25:23.676868+00:00",
"stop_seconds": 0.5348,
"exit_code": 0,
"left_stopped": true
},
"traffic": {
"seconds": 8.0006,
"concurrency": 4,
"successful_requests": 3857,
"failed_attempts": 0,
"successful_rps": 482.09,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 6.513,
"p95": 16.08,
"p99": 56.428
},
"instances": [
"118d370a438c4c14aabe117988bc86e9",
"6e358c3478904b0e97230b8ceeb17c24"
]
},
"post_stop_write_seconds_from_action_start": 1.2915,
"post_stop_write_seconds_from_stop_completion": 0.0308,
"read_during_outage": {
"status": 200,
"confirmed_message_preserved": true,
"seconds_from_action_start": 1.2842,
"seconds_from_stop_completion": 0.0235
},
"new_connection_after_stop": {
"accepted": false,
"error_type": "TimeoutError"
},
"stopped_remains_stopped": true
},
{
"stopped": {
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
"name": "mtusi-lab4-api-2",
"instance": "6e358c3478904b0e97230b8ceeb17c24",
"release": "lab4-v1",
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
"stopped_at_utc": "2026-11-02T09:25:37.448855+00:00",
"stop_seconds": 0.5935,
"exit_code": 0,
"left_stopped": true
},
"traffic": {
"seconds": 8.0018,
"concurrency": 4,
"successful_requests": 4137,
"failed_attempts": 1,
"successful_rps": 517.011,
"error_ratio": 0.000242,
"successful_latency_ms": {
"p50": 6.55,
"p95": 14.11,
"p99": 53.684
},
"instances": [
"69d3312a6f6b4f7bb50651c8632e196a",
"6e358c3478904b0e97230b8ceeb17c24"
]
},
"post_stop_write_seconds_from_action_start": 1.2502,
"post_stop_write_seconds_from_stop_completion": 0.0189,
"read_during_outage": {
"status": 200,
"confirmed_message_preserved": true,
"seconds_from_action_start": 1.2386,
"seconds_from_stop_completion": 0.0073
},
"new_connection_after_stop": {
"accepted": false,
"error_type": "TimeoutError"
},
"stopped_remains_stopped": true
}
],
"total": {
"seconds": 30.9392,
"concurrency": 4,
"successful_requests": 17688,
"failed_attempts": 1,
"successful_rps": 571.702,
"error_ratio": 5.7e-05,
"successful_latency_ms": {
"p50": 5.77,
"p95": 12.069,
"p99": 46.873
},
"instances": [
"118d370a438c4c14aabe117988bc86e9",
"427b43f714ff4bc3b99698dc90cafa3b",
"69d3312a6f6b4f7bb50651c8632e196a",
"6e358c3478904b0e97230b8ceeb17c24"
]
},
"data": {
"confirmed_unique_messages": 17688,
"history_messages": 17688,
"confirmed_preserved": true,
"duplicate_ids": 0,
"duplicate_texts": 0
},
"each_original_container_tested": true
}
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+14
View File
@@ -0,0 +1,14 @@
python3 scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
..................... [100%]
=============================== warnings summary ===============================
student-tests/test_attachments.py: 13 warnings
student-tests/test_idempotency.py: 1 warning
/usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
datetime_now = datetime.datetime.utcnow()
student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename
/usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack.
warnings.warn(
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
21 passed, 15 warnings in 4.15s
+28
View File
@@ -0,0 +1,28 @@
# Official public images/binaries were removed upstream; build fixed release sources.
FROM golang:1.24.2-bookworm AS minio-build
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local
RUN git clone --depth 1 --branch RELEASE.2025-04-22T22-12-26Z https://github.com/minio/minio.git /src
WORKDIR /src
RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/minio .
RUN test "$(git rev-parse HEAD)" = 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e' -o /out/minio .
FROM golang:1.24.2-bookworm AS mc-build
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local
RUN git clone --depth 1 --branch RELEASE.2025-04-16T18-13-26Z https://github.com/minio/mc.git /src
WORKDIR /src
RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/mc .
RUN test "$(git rev-parse HEAD)" = b00526b153a31b36767991a4f5ce2cced435ee8e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/mc/cmd.Version=2025-04-16T18:13:26Z -X github.com/minio/mc/cmd.ReleaseTag=RELEASE.2025-04-16T18-13-26Z -X github.com/minio/mc/cmd.CommitID=b00526b153a31b36767991a4f5ce2cced435ee8e' -o /out/mc .
FROM python:3.13.5-slim AS minio
LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.version="RELEASE.2025-04-22T22-12-26Z"
RUN useradd --uid 10002 --create-home minio && mkdir /data && chown 10002:10002 /data
COPY --from=minio-build /out/minio /usr/local/bin/minio
COPY infra/minio/server.sh /server.sh
USER 10002
ENTRYPOINT ["/bin/sh", "/server.sh"]
CMD ["server", "/data", "--address", ":9000", "--console-address", "127.0.0.1:9001"]
FROM debian:bookworm-slim AS mc
LABEL org.opencontainers.image.source="https://github.com/minio/mc" org.opencontainers.image.version="RELEASE.2025-04-16T18-13-26Z"
COPY --from=mc-build /out/mc /usr/local/bin/mc
ENTRYPOINT ["/bin/sh", "/config/bootstrap.sh"]
+8
View File
@@ -0,0 +1,8 @@
{
"Version": "2012-10-17",
"Statement": [
{"Effect":"Allow","Action":["s3:GetBucketLocation"],"Resource":["arn:aws:s3:::messenger-attachments"]},
{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::messenger-attachments"],"Condition":{"StringLike":{"s3:prefix":["originals/*"]}}},
{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::messenger-attachments/originals/*"]}
]
}
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
mc alias set local http://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null
mc mb --ignore-existing local/messenger-attachments >/dev/null
mc anonymous set none local/messenger-attachments >/dev/null
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null
mc admin policy create local messenger-api /config/api-policy.json >/dev/null
mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null
printf '%s\n' 'PASS: private bucket and restricted API policy ready (credentials hidden).'
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
export MINIO_ROOT_USER="$(cat /run/secrets/minio_root_user)"
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)"
exec minio "$@"
+62
View File
@@ -0,0 +1,62 @@
worker_processes 1;
error_log /dev/stderr warn;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
access_log off;
server_tokens off;
limit_conn_zone $server_name zone=uploads:64k;
# Docker DNS; resolve updates the upstream when replicas get new IPs.
resolver 127.0.0.11 valid=1s ipv6=off;
resolver_timeout 1s;
upstream messenger {
zone messenger 64k;
# All A records of api participate in the default round-robin balancing.
server api:8080 resolve max_fails=1 fail_timeout=2s;
}
server {
listen 8080;
server_name messenger;
location ~ ^/api/v1/chats/[^/]+/attachments$ {
limit_conn uploads 16;
client_max_body_size 11m;
client_body_buffer_size 64k;
client_body_timeout 10s;
proxy_request_buffering on;
proxy_pass http://messenger;
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header Connection "";
proxy_connect_timeout 1s;
proxy_read_timeout 30s;
proxy_send_timeout 30s;
proxy_next_upstream error timeout http_503;
proxy_next_upstream_tries 2;
}
location / {
proxy_pass http://messenger;
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header Connection "";
proxy_connect_timeout 1s;
proxy_read_timeout 5s;
proxy_send_timeout 5s;
# Passive checks. Do NOT enable non_idempotent: an already-sent POST
# must not be resent by Nginx after an ambiguous response.
proxy_next_upstream error timeout http_503;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 6s;
}
}
}
+34
View File
@@ -0,0 +1,34 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
CREATE INDEX chats_history_idx ON chats(created_at, id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
);
CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id);
CREATE TABLE sessions (
token_hash text PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id),
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
+11
View File
@@ -0,0 +1,11 @@
-- Additive: old lab2 tables and messages remain valid.
CREATE TABLE idempotency_records (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
request_hash text NOT NULL,
message_id uuid REFERENCES messages(id),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id, chat_id, key)
);
CREATE INDEX idempotency_expiry_idx ON idempotency_records(expires_at);
+23
View File
@@ -0,0 +1,23 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL REFERENCES users(id),
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX attachments_chat_idx ON attachments(chat_id,id);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id),
attachment_id uuid NOT NULL REFERENCES attachments(id),
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id,attachment_id),
UNIQUE (message_id,position)
);
+51
View File
@@ -0,0 +1,51 @@
annotated-types==0.8.0
anyio==4.9.0
argon2-cffi==25.1.0
argon2-cffi-bindings==26.1.0
attrs==26.1.0
boto3==1.38.27
botocore==1.38.46
certifi==2026.10.23
cffi==2.1.1
charset-normalizer==3.5.2
click==8.5.0
fastapi==0.115.12
h11==0.16.0
httpcore==1.0.9
httpx==0.28.1
idna==3.20
iniconfig==2.3.1
jmespath==1.1.0
jsonschema==4.26.0
jsonschema-path==0.3.4
jsonschema-specifications==2025.9.1
lazy-object-proxy==1.13.0
openapi-schema-validator==0.6.3
openapi-spec-validator==0.7.2
packaging==26.3
pathable==0.4.4
pillow==11.3.0
pluggy==1.7.0
psycopg==3.2.9
psycopg-binary==3.2.9
psycopg-pool==3.2.6
pycparser==3.12
pydantic==2.11.5
pydantic_core==2.33.2
Pygments==2.21.0
pytest==8.4.0
python-dateutil==2.9.0.post0
PyYAML==6.0.3
redis==6.2.0
referencing==0.36.2
requests==2.34.2
rfc3339-validator==0.1.4
rpds-py==2026.9.1
s3transfer==0.13.1
six==1.17.0
sniffio==1.3.1
starlette==0.46.2
typing-inspection==0.4.4
typing_extensions==4.16.0
urllib3==2.8.0
uvicorn==0.34.3
+12
View File
@@ -0,0 +1,12 @@
fastapi==0.115.12
uvicorn==0.34.3
psycopg[binary]==3.2.9
psycopg-pool==3.2.6
redis==6.2.0
argon2-cffi==25.1.0
pytest==8.4.0
httpx==0.28.1
pydantic==2.11.5
anyio==4.9.0
boto3==1.38.27
Pillow==11.3.0
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env python3
"""Container-side privileged test utilities. Input over stdin; never output credentials."""
import base64
from datetime import datetime,timezone
import io
import json
from pathlib import Path
import socket
import sys
import uuid
import boto3
from botocore import UNSIGNED
from botocore.config import Config
from botocore.exceptions import ClientError
import psycopg
from psycopg import sql
from psycopg.rows import dict_row
from messenger.db import setting,Database
from messenger.object_store import ObjectStore
from messenger.cleanup import sweep
def main():
p=json.load(sys.stdin)
action=p['action']
store=ObjectStore()
result={}
with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row) as conn:
if action=='lookup':
result=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(p['id'],)).fetchone()
elif action=='counts':
result={'attachments':conn.execute('SELECT count(*) AS n FROM attachments').fetchone()['n'],
'objects':sum(len(page.get('Contents',[])) for page in store.pages())}
elif action=='list':
result={'keys':[obj['Key'] for page in store.pages() for obj in page.get('Contents',[])]}
elif action=='put':
raw=base64.b64decode(p['bytes'])
store.put(p['key'],io.BytesIO(raw),len(raw),'image/png')
result={'created':True}
elif action=='delete':
store.delete(p['key'])
result={'deleted':True}
elif action=='exists':
try:
obj=store.client.head_object(Bucket=store.bucket,Key=p['key'])
result={'exists':True,'length':obj['ContentLength']}
except ClientError as exc:
if exc.response['ResponseMetadata']['HTTPStatusCode']!=404:
raise
result={'exists':False}
elif action=='tracked':
result={'tracked':conn.execute('SELECT count(*) AS n FROM attachments WHERE object_key=%s',(p['key'],)).fetchone()['n']}
elif action=='security':
unsigned=boto3.client('s3',endpoint_url=setting('S3_ENDPOINT'),region_name='us-east-1',config=Config(signature_version=UNSIGNED,s3={'addressing_style':'path'},connect_timeout=1,read_timeout=3,retries={'total_max_attempts':1}))
try:
unsigned.get_object(Bucket=store.bucket,Key=p['key'])
anonymous=200
except ClientError as exc:
anonymous=exc.response['ResponseMetadata']['HTTPStatusCode']
try:
store.client.create_bucket(Bucket='forbidden-'+uuid.uuid4().hex)
create_bucket=200
except ClientError as exc:
create_bucket=exc.response['ResponseMetadata']['HTTPStatusCode']
try:
store.client.list_objects_v2(Bucket=store.bucket,Prefix='forbidden/')
other_prefix=200
except ClientError as exc:
other_prefix=exc.response['ResponseMetadata']['HTTPStatusCode']
try:
sock=socket.create_connection(('minio',9001),timeout=1)
sock.close()
console=True
except OSError:
console=False
result={'anonymous_get':anonymous,'create_bucket_with_api_credentials':create_bucket,
'list_other_prefix':other_prefix,'console_reachable_from_api':console,
'root_secret_mounted_in_api':Path('/run/secrets/minio_root_password').exists()}
unsigned.close()
elif action=='install':
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename={} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['filename'])))
conn.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
result={'installed':True}
elif action=='remove':
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON attachments').format(sql.Identifier(p['trigger'])))
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
result={'removed':True}
elif action=='waiting':
result={'waiting':conn.execute("SELECT count(*) AS n FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO attachments%%'").fetchone()['n']}
elif action=='sweep':
db=Database()
db.open()
try:
result=sweep(db,store,grace_seconds=p.get('grace',60),apply=p.get('apply',False))
finally:
db.close()
else:
raise ValueError('Unknown check action')
store.close()
print(json.dumps(result,default=str))
if __name__=='__main__':
main()
+32
View File
@@ -0,0 +1,32 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets
python3 - <<'PY'
from pathlib import Path
import secrets
root = Path('secrets')
p = root / 'postgres_password'
if not p.exists():
p.write_text(secrets.token_hex(24))
pw = p.read_text().strip()
for name, value in {
'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger',
'cache_url': 'redis://redis:6379/0',
'minio_root_user': 'lab4-admin',
'minio_root_password': secrets.token_hex(24),
's3_access_key_id': 'messenger-api',
's3_secret_access_key': secrets.token_hex(24),
}.items():
path = root / name
if not path.exists():
path.write_text(value)
# Bind-mounted Docker secrets must be readable by container uid 10001.
# Parent directory remains 0700 on the host.
for path in root.iterdir():
path.chmod(0o444)
if not Path('.env').exists():
Path('.env').write_text('SESSION_TTL_SECONDS=3600\n')
print('Local secrets ready (values hidden).')
PY
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""Capture repeatable short load run and sanitized environment/resource evidence."""
import datetime
import json
import platform
import re
import subprocess
import sys
import time
import cluster
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def output(args):
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
def counts():
code = "import json,psycopg; from messenger.db import setting; c=psycopg.connect(setting('DATABASE_URL')); print(json.dumps({t:c.execute('SELECT count(*) FROM '+t).fetchone()[0] for t in ['users','chats','messages','sessions','idempotency_records']})); c.close()"
return json.loads(output(['docker','exec',cluster.container_ids('api')[0],'python','-c',code]))
def resource_summary(samples):
peaks = {}
for sample in samples:
for container in sample['containers']:
peak = peaks.setdefault(container['Name'], {'cpu_percent_peak':0, 'memory_mib_peak':0})
peak['cpu_percent_peak'] = max(peak['cpu_percent_peak'],float(container['CPUPerc'].rstrip('%')))
value,unit = re.match(r'([0-9.]+)([A-Za-z]+)',container['MemUsage']).groups()
multiplier = {'GiB':1024,'MiB':1,'KiB':1/1024,'B':1/1048576}[unit]
peak['memory_mib_peak'] = round(max(peak['memory_mib_peak'],float(value)*multiplier),3)
return {'samples':len(samples),'per_container_peaks':peaks}
def main():
names = cluster.container_ids()
environment = {
'captured_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(),
'host_os': platform.platform(), 'host_architecture': platform.machine(),
'docker': json.loads(output(['docker','info','--format','{"server_version":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"architecture":"{{.Architecture}}"}'])),
'python_in_image': output(['docker','exec',cluster.container_ids('api')[0],'python','--version']),
'dataset_before': counts(),
'limits': {'api_each':'1 CPU / 512 MiB','postgres':'1 CPU / 512 MiB','redis':'0.25 CPU / 128 MiB','lb':'0.5 CPU / 128 MiB'},
'warmup': 'API already healthy; acceptance, persistence and dependency checks performed before benchmark. No separate timed load warmup.',
'replicas':len(cluster.replicas()), 'repetitions': 1, 'duration_seconds':30, 'concurrency':4,
}
if sys.platform == 'darwin':
environment['host_cpu'] = output(['sysctl','-n','machdep.cpu.brand_string'])
environment['host_memory_bytes'] = int(output(['sysctl','-n','hw.memsize']))
samples = []
with (ROOT/'evidence/load-output.txt').open('w') as log:
proc = subprocess.Popen([sys.executable,'tests/load.py','--duration','30','--concurrency','4','--output','evidence/load.json'],cwd=ROOT,stdout=log,stderr=subprocess.STDOUT)
while proc.poll() is None:
rows = output(['docker','stats','--no-stream','--format','{{json .}}',*names])
samples.append({'at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
'containers':[{k:v for k,v in json.loads(row).items() if k in ('Name','CPUPerc','MemUsage','MemPerc','PIDs')}
for row in rows.splitlines()]})
time.sleep(1)
if proc.returncode:
raise SystemExit(proc.returncode)
environment['dataset_after'] = counts()
(ROOT/'evidence/environment.json').write_text(json.dumps(environment,ensure_ascii=False,indent=2)+'\n')
(ROOT/'evidence/resources.json').write_text(json.dumps(samples,indent=2)+'\n')
(ROOT/'evidence/resource-summary.json').write_text(json.dumps(resource_summary(samples),indent=2)+'\n')
print((ROOT/'evidence/load.json').read_text())
if __name__ == '__main__':
main()
+214
View File
@@ -0,0 +1,214 @@
#!/usr/bin/env python3
"""Ten overlapping PNG uploads with kernel-recorded memory peaks; stdlib only.
Recreates API containers (never volumes) to reset memory high-water marks.
Run from any directory: python3 scripts/capture_uploads.py.
"""
import concurrent.futures
import datetime
import hashlib
import json
import platform
import struct
import subprocess
import sys
import threading
import time
import urllib.parse
import zlib
from pathlib import Path
import cluster
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check # noqa: E402
WORKERS = 10
WIDTH, HEIGHT = 3000, 1000
EVIDENCE = ROOT / 'evidence/concurrent-uploads.json'
def make_png():
"""Stable, incompressible RGB rows without a host image library."""
def chunk(kind, data):
return (struct.pack('!I', len(data)) + kind + data
+ struct.pack('!I', zlib.crc32(kind + data) & 0xffffffff))
compressor = zlib.compressobj(level=0)
compressed = []
for row in range(HEIGHT):
pixels = hashlib.shake_256(b'lab4-concurrent-upload-v1:' + str(row).encode()).digest(WIDTH * 3)
compressed.append(compressor.compress(b'\x00' + pixels))
compressed.append(compressor.flush())
return (b'\x89PNG\r\n\x1a\n'
+ chunk(b'IHDR', struct.pack('!IIBBBBB', WIDTH, HEIGHT, 8, 2, 0, 0, 0))
+ chunk(b'IDAT', b''.join(compressed)) + chunk(b'IEND', b''))
def memory(container):
# cgroup peaks include allocations between observations; VmHWM is PID 1's
# resident peak, whereas cgroup memory also includes file cache and children.
code = '''import json,pathlib
def number(path):
return int(pathlib.Path(path).read_text().strip())
root=pathlib.Path('/sys/fs/cgroup')
if (root/'memory.peak').exists():
source='cgroup-v2'
peak=number(root/'memory.peak'); current=number(root/'memory.current')
events=dict((k,int(v)) for k,v in (line.split() for line in (root/'memory.events').read_text().splitlines()))
else:
root=root/'memory'; source='cgroup-v1'
peak=number(root/'memory.max_usage_in_bytes'); current=number(root/'memory.usage_in_bytes')
events={'failcnt':number(root/'memory.failcnt')}
status=dict(line.split(':',1) for line in pathlib.Path('/proc/1/status').read_text().splitlines() if ':' in line)
print(json.dumps({'source':source,'cgroup_peak_bytes':peak,'cgroup_current_bytes':current,
'pid1_vmhwm_bytes':int(status['VmHWM'].split()[0])*1024,
'pid1_vmrss_bytes':int(status['VmRSS'].split()[0])*1024,'memory_events':events}))
'''
return json.loads(cluster.command(['docker', 'exec', container, 'python', '-c', code]))
def version_context():
revision = cluster.command(['git', 'rev-parse', 'HEAD'])
dirty = subprocess.run(['git', 'diff', '--quiet'], cwd=ROOT, check=False).returncode != 0
address = urllib.parse.urlsplit(cluster.BASE_URL)
return {
'git_commit': revision, 'tracked_worktree_dirty': dirty,
'host_python': platform.python_version(), 'host_os': platform.platform(),
'docker_version': cluster.command(['docker', 'version', '--format', '{{.Server.Version}}']),
'compose_version': cluster.compose('version', '--short'),
'contract_version': json.loads((ROOT / 'lab.json').read_text())['contract_version'],
'endpoint': {'scheme': address.scheme, 'host': address.hostname, 'port': address.port},
}
def experiment(evidence):
payload = make_png()
check(9_000_000 <= len(payload) <= 10 * 1024 * 1024, 'generated PNG size outside experiment bounds')
digest = hashlib.sha256(payload).hexdigest()
evidence.update({
'context': version_context(),
'input': {'format': 'PNG', 'mode': 'RGB', 'width': WIDTH, 'height': HEIGHT,
'size_bytes': len(payload), 'sha256': digest,
'generator': 'SHAKE-256 lab4-concurrent-upload-v1 row seeds; filter 0; zlib level 0',
'same_immutable_bytes_for_all_requests': True},
'concurrency': WORKERS,
'reset': {'command': 'docker compose up -d --no-deps --force-recreate api',
'purpose': 'Reset API cgroup memory.peak and PID 1 VmHWM; retain all volumes'},
'peak_scope': 'Fresh API startup, bootstrap, ten uploads and representative download; no peak sampling gaps',
})
cluster.compose('up', '-d', '--no-deps', '--force-recreate', 'api')
containers = cluster.container_ids('api')
check(len(containers) == 2, 'experiment requires exactly two API containers')
for container in containers:
cluster.wait_healthy(container)
cluster.wait_cluster(2)
replicas = cluster.replicas()
evidence['cluster_before'] = cluster.snapshot()
evidence['baseline'] = {item['instance']: memory(item['container']) for item in replicas}
limits = [item for item in evidence['cluster_before'] if item['service'] == 'api']
evidence['memory_limits'] = [
{key: item[key] for key in ('instance', 'memory_bytes', 'nano_cpus')} for item in limits]
check(all(item['memory_bytes'] == 512 * 1024 * 1024 and item['nano_cpus'] == 1_000_000_000
for item in limits), 'API limits must be 512 MiB and 1 CPU each')
client = Client(timeout=180)
alice, bob, _eve, chat = client.bootstrap()
barrier = threading.Barrier(WORKERS, timeout=30)
lock = threading.Lock()
activity = {'active': 0, 'peak': 0}
origin = time.monotonic()
def upload(index):
worker = Client(timeout=180)
barrier.wait()
with lock:
start = time.monotonic() - origin
activity['active'] += 1
activity['peak'] = max(activity['peak'], activity['active'])
item = {'index': index, 'start_seconds': start}
attachment = None
try:
response = worker.call(
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=concurrent-%02d.png' % index,
expected=201, actor=alice, raw=payload,
headers={'Content-Type': 'image/png'}, schema='Attachment')
attachment = response['json']
item.update({'status': response['status'], 'instance': response['headers'].get('X-Instance-Id')})
check(attachment['sha256'] == digest and attachment['size_bytes'] == len(payload),
'uploaded original checksum/size mismatch')
check(attachment['chat_id'] == chat['id'] and attachment['owner_id'] == alice['id'],
'uploaded attachment ownership mismatch')
check(attachment['status'] == 'ready', 'uploaded attachment not ready')
check(attachment['content_type'] == 'image/png', 'uploaded MIME type mismatch')
item.update({'attachment_id': attachment['id'], 'metadata_verified': True})
except Failure as exc:
item['error'] = str(exc)
except Exception as exc:
# Never serialize arbitrary library exception text, request or actor.
item['error'] = type(exc).__name__
finally:
with lock:
item['end_seconds'] = time.monotonic() - origin
activity['active'] -= 1
return item, attachment
with concurrent.futures.ThreadPoolExecutor(max_workers=WORKERS) as pool:
completed = list(pool.map(upload, range(WORKERS)))
requests = [item for item, _attachment in completed]
evidence['requests'] = requests
evidence['elapsed_seconds'] = time.monotonic() - origin
evidence['peak_active_requests'] = activity['peak']
overlap = min(item['end_seconds'] for item in requests) - max(item['start_seconds'] for item in requests)
evidence['all_ten_overlap_seconds'] = max(0, overlap)
evidence['all_ten_overlapping'] = overlap > 0
evidence['successful_count'] = sum('error' not in item for item in requests)
evidence['error_count'] = WORKERS - evidence['successful_count']
evidence['observed_instances'] = sorted({item['instance'] for item in requests if item.get('instance')})
successful = [attachment for item, attachment in completed if 'error' not in item]
if successful:
download = client.call('GET', '/api/v1/attachments/' + successful[0]['id'] + '/content', actor=bob)
check(download['bytes'] == payload, 'representative download differs from original')
evidence['representative_download'] = {
'verified_as': 'bob (chat member)', 'matches_original_bytes': True,
'size_bytes': len(download['bytes']), 'sha256': hashlib.sha256(download['bytes']).hexdigest()}
evidence['final'] = {item['instance']: memory(item['container']) for item in replicas}
evidence['cluster_after'] = cluster.snapshot()
states = cluster.inspect(containers)
evidence['oom_killed'] = {item['Name'].lstrip('/'): item['State']['OOMKilled'] for item in states}
evidence['per_node_peaks'] = evidence['final']
check(evidence['successful_count'] == WORKERS, 'not all ten uploads succeeded')
check(len(evidence['observed_instances']) >= 2, 'uploads did not reach both API instances')
check(overlap > 0 and activity['peak'] == WORKERS, 'ten requests did not all overlap')
check(not any(evidence['oom_killed'].values()), 'an API container was OOM-killed')
for item in replicas:
before, after = evidence['baseline'][item['instance']], evidence['final'][item['instance']]
check(after['cgroup_peak_bytes'] <= 512 * 1024 * 1024, 'API exceeded configured memory limit')
for event in ('oom', 'oom_kill', 'failcnt'):
check(before['memory_events'].get(event, 0) == 0 and after['memory_events'].get(event, 0) == 0,
'API recorded an OOM or cgroup allocation failure')
check(all(item['State']['Running'] and item['State'].get('Health', {}).get('Status') == 'healthy'
for item in states), 'both APIs must remain running and healthy')
cluster.wait_cluster(2)
def main():
evidence = {'captured_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'result': 'FAIL'}
try:
experiment(evidence)
evidence['result'] = 'PASS'
except Failure as exc:
evidence['failure'] = str(exc)
except Exception as exc:
evidence['failure'] = type(exc).__name__
finally:
EVIDENCE.parent.mkdir(parents=True, exist_ok=True)
EVIDENCE.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
print(evidence['result'] + ': evidence/concurrent-uploads.json')
if evidence['result'] != 'PASS':
print(evidence.get('failure', 'experiment failed'))
raise SystemExit(1)
if __name__ == '__main__':
main()
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
# A unique disposable Compose project; never touches normal project volumes.
set -eu
cd "$(dirname "$0")/.."
project="mtusi-lab4-migration-check-$$"
if [ -n "$(docker compose -p "$project" ps -aq)" ]; then
echo 'Refusing to reuse a pre-existing scratch project' >&2
exit 1
fi
cleanup() {
docker compose -p "$project" down --volumes >/dev/null 2>&1
}
trap cleanup EXIT INT TERM
docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis minio minio-init
docker compose -p "$project" run --rm --no-deps api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache'
echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.'
+196
View File
@@ -0,0 +1,196 @@
#!/usr/bin/env python3
"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence."""
import argparse
import datetime
import json
import os
import re
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
def command(args, timeout=120):
return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip()
def compose(*args, timeout=120):
return command(['docker', 'compose', *args], timeout=timeout)
def inspect(containers):
if not containers:
return []
return json.loads(command(['docker','inspect',*containers]))
def container_ids(service=None, all_containers=False):
args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else [])
return compose(*args).split()
def probe(container, host='127.0.0.1', path='/health/live'):
code = '''import json,sys,urllib.request,urllib.error
try:
r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4)
except urllib.error.HTTPError as exc:
r=exc
print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')}))
'''
return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8))
def replicas():
items = []
for data in inspect(container_ids('api')):
if data['State']['Running']:
result = probe(data['Id'])
items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'),
'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']})
return items
def snapshot():
items = []
for data in inspect(container_ids(all_containers=True)):
service = data['Config']['Labels']['com.docker.compose.service']
item = {
'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'),
'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'),
'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'],
'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'],
'published_ports':data['HostConfig']['PortBindings'],
'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'),
'networks':sorted(data['NetworkSettings']['Networks']),
}
if service == 'api' and data['State']['Running']:
result = probe(data['Id'])
item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']})
items.append(item)
return items
def wait_healthy(container, timeout=60):
deadline = time.monotonic()+timeout
while time.monotonic() < deadline:
data = inspect([container])[0]
if data['State'].get('Health',{}).get('Status') == 'healthy':
return
time.sleep(0.3)
raise RuntimeError('Container did not become healthy')
def wait_cluster(count=2, timeout=60):
deadline = time.monotonic()+timeout
seen = set()
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response:
alias = response.headers.get('X-Instance-Id')
if alias:
seen.add(alias)
if len(seen) >= count:
return sorted(seen)
except (OSError, urllib.error.URLError):
pass
time.sleep(0.1)
raise RuntimeError('Load balancer did not discover required API instances')
def stop_one(target):
if not re.fullmatch(r'[a-f0-9]{32}', target):
raise ValueError('Invalid target instance alias')
matching = [item for item in replicas() if item['instance']==target]
if len(matching) != 1:
raise RuntimeError('Target must match exactly one running API process')
item = matching[0]
started = time.monotonic()
command(['docker','stop','--time','7',item['container']],timeout=9)
state = inspect([item['container']])[0]['State']
if state['Running'] or state['ExitCode'] != 0:
raise RuntimeError('API did not stop gracefully')
result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True}
(ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n')
return result
def restart():
compose('restart','api','postgres','redis','minio','cleanup','lb',timeout=30)
compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40)
return wait_cluster(2,timeout=15)
def front_address(container):
networks = inspect([container])[0]['NetworkSettings']['Networks']
return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front'))
def replace_one(container):
before = set(container_ids('api'))
if container not in before or len(before) != 2:
raise RuntimeError('Replacement expects exactly two running API containers')
old_address = front_address(container)
# Start-first ensures a genuinely different IP while the old address is still occupied.
compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api')
expanded = set(container_ids('api'))
added = expanded-before
if len(added) != 1 or not before.issubset(expanded):
raise RuntimeError('Start-first step unexpectedly changed an existing container')
replacement = added.pop()
wait_healthy(replacement)
new_address = front_address(replacement)
if new_address == old_address:
raise RuntimeError('Replacement did not get a different IP')
expected_three = {item['instance'] for item in replicas()}
if set(wait_cluster(3)) != expected_three:
raise RuntimeError('Load balancer did not discover all three live addresses')
command(['docker','stop','--time','7',container],timeout=10)
state = inspect([container])[0]['State']
if state['ExitCode'] != 0:
raise RuntimeError('Old release did not terminate gracefully')
command(['docker','rm',container])
compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api')
after = set(container_ids('api'))
if after != (before-{container}) | {replacement}:
raise RuntimeError('Sequential replacement changed the wrong containers')
expected = {item['instance'] for item in replicas()}
if set(wait_cluster(2)) != expected:
raise RuntimeError('Unexpected final backend aliases')
return {'old_container':container,'new_container':replacement,
'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True,
'instances_during_start_first':sorted(expected_three),
'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']}
def main():
parser = argparse.ArgumentParser()
parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec'])
parser.add_argument('arguments',nargs=argparse.REMAINDER)
args = parser.parse_args()
if args.action == 'wait':
print(json.dumps({'instances':wait_cluster()}))
elif args.action == 'snapshot':
print(json.dumps(snapshot(),indent=2))
elif args.action == 'stop-one':
print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID'])))
elif args.action == 'exec':
ids = container_ids('api')
if not ids or not args.arguments:
raise RuntimeError('A live API container and command are required')
raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT))
elif args.action == 'restart':
print(json.dumps({'instances':restart()}))
else:
compose('run','--rm','--no-deps','migrate')
print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2))
if __name__ == '__main__':
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/cluster.py restart
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
exec python3 scripts/cluster.py stop-one
+250
View File
@@ -0,0 +1,250 @@
#!/usr/bin/env python3
"""Grade4 attachment evidence with real S3 faults, process crash and grace cleanup."""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import os
from pathlib import Path
import socket
import subprocess
import sys
import time
import urllib.request
import urllib.error
import urllib.parse
import uuid
import cluster
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client,Failure,check
def save(name,value):
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
print('PASS:',name,flush=True)
def internal(data,container=None):
container=container or cluster.container_ids('api')[0]
p=subprocess.run(['docker','exec','-i',container,'python','scripts/attachment_checks.py'],cwd=ROOT,
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=30)
return json.loads(p.stdout)
def upload(c,actor,chat,raw,filename='sample.png'):
return c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+urllib.parse.quote(filename),
201,actor=actor,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')
def private_bucket():
c=Client()
alice,bob,_,chat=c.bootstrap()
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
response=upload(c,alice,chat,raw)
att=response['json']
key=internal({'action':'lookup','id':att['id']})['object_key']
result=internal({'action':'security','key':key})
check(result['anonymous_get']==403,'private object is accessible without S3 authentication')
check(result['create_bucket_with_api_credentials']==403 and result['list_other_prefix']==403,'API S3 credentials are too broad')
check(not result['console_reachable_from_api'] and not result['root_secret_mounted_in_api'],'root/console exposed to API')
before=internal({'action':'counts'})
output=cluster.compose('run','--rm','--no-deps','minio-init')
check(internal({'action':'counts'})==before,'repeated bucket bootstrap changed data')
download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
check(download['bytes']==raw,'bootstrap changed original bytes')
save('s3-security.json',{'result':'PASS',**result,'bootstrap_repeated_without_data_loss':True,'bootstrap_output':output})
def upload_failover():
cluster.wait_cluster()
c=Client()
alice,bob,_,chat=c.bootstrap()
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
response=upload(c,alice,chat,raw)
att=response['json']
alias=response['headers']['X-Instance-Id']
c.message(alice,chat,'',attachments=[att['id']])
target=next(node for node in cluster.replicas() if node['instance']==alias)
started=time.monotonic()
try:
stopped=cluster.stop_one(alias)
deadline=started+10
while True:
try:
download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
break
except Failure:
check(time.monotonic()<deadline,'file download did not recover in10s')
time.sleep(.1)
check(download['headers']['X-Instance-Id']!=alias,'download came from stopped process')
check(download['bytes']==raw and hashlib.sha256(download['bytes']).hexdigest()==att['sha256'],'file changed across replicas')
check(not cluster.inspect([target['container']])[0]['State']['Running'],'target was not left stopped')
result={'result':'PASS','stopped':stopped,'download_instance':download['headers']['X-Instance-Id'],
'same_bytes':True,'same_sha256':True,'seconds_from_action_start':round(time.monotonic()-started,4)}
finally:
cluster.command(['docker','start',target['container']])
cluster.wait_healthy(target['container'])
cluster.wait_cluster()
save('attachment-failover.json',result)
def interrupted_upload():
c=Client()
alice,_,_,chat=c.bootstrap()
before=internal({'action':'counts'})
parsed=urllib.parse.urlsplit(c.base)
sock=socket.create_connection((parsed.hostname,parsed.port or 80),timeout=3)
path='/api/v1/chats/'+chat['id']+'/attachments?filename=interrupted.png'
headers=('POST '+path+' HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer '+alice['token']+
'\r\nContent-Type: image/png\r\nContent-Length: 500000\r\nConnection: close\r\n\r\n')
sock.sendall(headers.encode()+b'\x89PNG\r\n\x1a\n'+b'x'*100)
sock.shutdown(socket.SHUT_RDWR)
sock.close()
time.sleep(.5)
after=internal({'action':'counts'})
check(before==after,'interrupted request left an object or metadata')
save('interrupted-upload.json',{'result':'PASS','declared_bytes':500000,'sent_bytes':108,
'metadata_unchanged':True,'s3_object_count_unchanged':True})
def crash_orphan():
cluster.wait_cluster()
c=Client()
alice,bob,_,chat=c.bootstrap()
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
nodes=cluster.replicas()
target,survivor=nodes[0],nodes[1]
filename='crash_'+uuid.uuid4().hex+'.png'
fixture={'filename':filename,'function':'crash_'+uuid.uuid4().hex,'trigger':'crash_'+uuid.uuid4().hex}
before=set(internal({'action':'list'},survivor['container'])['keys'])
internal({'action':'install',**fixture},survivor['container'])
code='''import json,base64,sys,urllib.request,urllib.error
p=json.load(sys.stdin)
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=base64.b64decode(p['bytes']),method='POST',headers={'Authorization':'Bearer '+p['token'],'Content-Type':'image/png'})
try:
response=urllib.request.urlopen(r,timeout=8)
print(json.dumps({'status':response.code}))
except Exception as exc:
print(json.dumps({'status':None,'error_type':type(exc).__name__}))
'''
data={'address':cluster.front_address(target['container']),'path':'/api/v1/chats/'+chat['id']+'/attachments?filename='+filename,
'bytes':base64.b64encode(raw).decode(),'token':alice['token']}
def post():
p=subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,input=json.dumps(data),capture_output=True,text=True,check=True,timeout=12)
return json.loads(p.stdout)
pool=ThreadPoolExecutor(max_workers=1)
future=pool.submit(post)
try:
deadline=time.monotonic()+5
while not internal({'action':'waiting'},survivor['container'])['waiting']:
check(time.monotonic()<deadline and not future.done(),'did not observe PUT followed by pending metadata INSERT')
time.sleep(.05)
after=set(internal({'action':'list'},survivor['container'])['keys'])
created=after-before
check(len(created)==1,'expected exactly one new S3 object during pending metadata INSERT')
orphan=created.pop()
cluster.command(['docker','kill','--signal','SIGKILL',target['container']])
crashed=cluster.inspect([target['container']])[0]['State']
check(not crashed['Running'] and crashed['ExitCode']==137,'SIGKILL did not produce the expected stopped state')
observed_exit_code=crashed['ExitCode']
response=future.result(timeout=12)
check(response['status']!=201,'crashed request returned an acknowledgement')
internal({'action':'remove',**fixture},survivor['container'])
check(internal({'action':'tracked','key':orphan},survivor['container'])['tracked']==0,'metadata unexpectedly committed after crash')
check(internal({'action':'exists','key':orphan},survivor['container'])['exists'],'crash did not leave an orphan to clean')
finally:
pool.shutdown(wait=True)
internal({'action':'remove',**fixture},survivor['container'])
cluster.command(['docker','start',target['container']])
cluster.wait_healthy(target['container'])
cluster.wait_cluster()
# A real orphan from crash, plus an old locked object and a tracked/linked attachment.
locked='originals/'+uuid.uuid4().hex
internal({'action':'put','key':locked,'bytes':base64.b64encode(raw).decode()})
tracked=upload(c,alice,chat,raw)['json']
c.message(alice,chat,'',attachments=[tracked['id']])
print('Waiting61s for the enforced60s cleanup grace (no time bypass).',flush=True)
time.sleep(61)
fresh='originals/'+uuid.uuid4().hex
internal({'action':'put','key':fresh,'bytes':base64.b64encode(raw).decode()})
lock_code='''import json,sys,psycopg
from messenger.db import setting
key=sys.stdin.readline().strip()
with psycopg.connect(setting('DATABASE_URL')) as conn:
conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(key,))
print('locked',flush=True)
sys.stdin.readline()
'''
holder=subprocess.Popen(['docker','exec','-i',survivor['container'],'python','-u','-c',lock_code],cwd=ROOT,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
try:
holder.stdin.write(locked+'\n')
holder.stdin.flush()
check(holder.stdout.readline().strip()=='locked','failed to hold cleanup coordination lock')
dry=internal({'action':'sweep','grace':60,'apply':False})
check(dry['deleted']==0 and dry['orphans']>=1 and dry['locked']>=1,'dry-run/grace/lock selection failed')
check(internal({'action':'exists','key':orphan})['exists'],'dry-run deleted an object')
applied=internal({'action':'sweep','grace':60,'apply':True})
check(applied['deleted']>=1,'orphan not removed after grace')
check(not internal({'action':'exists','key':orphan})['exists'],'crash orphan remained after cleanup')
check(internal({'action':'exists','key':locked})['exists'],'active upload lock was ignored')
check(internal({'action':'exists','key':fresh})['exists'],'fresh object was deleted')
repeat=internal({'action':'sweep','grace':60,'apply':True})
check(repeat['deleted']==0,'repeated sweep was not idempotent')
check(c.call('GET','/api/v1/attachments/'+tracked['id']+'/content',actor=bob)['bytes']==raw,'tracked/linked file was deleted')
finally:
holder.stdin.write('\n')
holder.stdin.flush()
holder.stdin.close()
holder.wait(timeout=5)
released=internal({'action':'sweep','grace':60,'apply':True})
check(not internal({'action':'exists','key':locked})['exists'],'unlocked old orphan not removed')
internal({'action':'delete','key':fresh})
save('orphan-cleanup.json',{'result':'PASS','fault':'SIGKILL API after successful S3PUT, before metadata COMMIT',
'crash_exit_code':observed_exit_code,'target_container':target['container'],'upload_acknowledged':False,'orphan_confirmed':True,'grace_wait_seconds':61,
'dry_run':dry,'apply':applied,'repeat':repeat,'after_lock_release':released,
'fresh_preserved':True,'tracked_and_linked_preserved':True,'active_lock_preserved':True})
def s3_outage():
c=Client(timeout=8)
alice,bob,_,chat=c.bootstrap()
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
att=upload(c,alice,chat,raw)['json']
try:
cluster.compose('stop','-t','2','minio')
req=urllib.request.Request(c.base+'/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',data=raw,method='POST',headers={'Authorization':'Bearer '+alice['token'],'Content-Type':'image/png'})
started=time.monotonic()
try:
response=urllib.request.urlopen(req,timeout=8)
except urllib.error.HTTPError as exc:
response=exc
with response:
status=response.code
elapsed=time.monotonic()-started
check(status in (502,503,504) and elapsed<8,'S3 outage was not bounded/safe')
finally:
cluster.compose('start','minio')
cluster.wait_healthy(cluster.container_ids('minio')[0])
cluster.wait_cluster()
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==raw,'original lost after S3 restart')
save('s3-outage.json',{'result':'PASS','upload_status_during_outage':status,'response_seconds':round(elapsed,4),
'original_preserved_after_restart':True,'same_session_after_recovery':True})
def main():
parser=argparse.ArgumentParser()
parser.add_argument('mode',choices=['private','failover','interrupted','cleanup','outage','all'],default='all',nargs='?')
args=parser.parse_args()
modes={'private':private_bucket,'failover':upload_failover,'interrupted':interrupted_upload,'cleanup':crash_orphan,'outage':s3_outage}
if args.mode=='all':
for function in modes.values():
function()
else:
modes[args.mode]()
if __name__=='__main__':
main()
+443
View File
@@ -0,0 +1,443 @@
#!/usr/bin/env python3
"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts."""
import argparse
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
import uuid
import cluster
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client, Failure, check
from resilience import history
def save(name, value):
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
print('PASS:', name)
def percentiles(values):
values = sorted(values)
return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None
for name,p in [('p50',.50),('p95',.95),('p99',.99)]}
class Traffic:
def __init__(self, concurrency=4):
self.c = Client(timeout=6)
self.alice,self.bob,_,self.chat = self.c.bootstrap()
self.concurrency = concurrency
self.done = threading.Event()
self.lock = threading.Lock()
self.attempts = []
self.confirmed = {}
self.prefix = uuid.uuid4().hex
def worker(self, number):
client = Client(timeout=6)
index = 0
while not self.done.is_set():
text = self.prefix+'-'+str(number)+'-'+str(index)
key = self.prefix+'-'+str(number)+'-'+str(index)
started = time.monotonic()
try:
response = client.message(self.alice,self.chat,text,key=key)
row = response['json']
attempt = {'started':started,'ended':time.monotonic(),'ok':True,
'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']}
with self.lock:
self.confirmed[row['id']] = row
index += 1
except Failure:
# Only idempotent message POST is retried, with its original body and key.
attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None}
time.sleep(0.02)
with self.lock:
self.attempts.append(attempt)
def start(self):
self.started = time.monotonic()
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)]
return self
def stop(self):
self.done.set()
for future in self.futures:
future.result(timeout=10)
self.pool.shutdown()
self.ended = time.monotonic()
def summary(self, start=None, end=None):
start = self.started if start is None else start
end = getattr(self,'ended',time.monotonic()) if end is None else end
with self.lock:
attempts = [a for a in self.attempts if start <= a['started'] < end]
success = [a for a in attempts if a['ok']]
duration = end-start
return {'seconds':round(duration,4),'concurrency':self.concurrency,
'successful_requests':len(success),'failed_attempts':len(attempts)-len(success),
'successful_rps':round(len(success)/duration,3),
'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0,
'successful_latency_ms':percentiles([a['latency'] for a in success]),
'instances':sorted({a['instance'] for a in success})}
def verify_data(self):
rows = history(self.c,self.bob,self.chat)
actual = {row['id']:row for row in rows}
check(len(actual)==len(rows),'duplicated message ids')
check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message')
check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed')
return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows),
'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0}
def concurrent_replay():
cluster.wait_cluster()
c = Client()
alice,bob,_,chat = c.bootstrap()
key = uuid.uuid4().hex
barrier = threading.Barrier(8)
def send(_):
local = Client()
barrier.wait()
return local.message(alice,chat,'Один результат через разные процессы',key=key)
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(send,range(8)))
instances = {r['headers']['X-Instance-Id'] for r in results}
check(len(instances)>=2,'concurrent retry did not reach different API processes')
check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages')
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
body={'text':'Другой body'},headers={'Idempotency-Key':key})
check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates')
save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances),
'responses_201':8,'messages_created':1,'different_body_status':409})
def verify_persistence():
before = cluster.snapshot()
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'],
cwd=ROOT,check=True,capture_output=True,text=True,timeout=190)
after = cluster.snapshot()
roles = ['api','postgres','redis','minio','cleanup','lb']
previous = {item['container']:item for item in before}
for item in after:
if item['service'] in roles:
old = previous[item['container']]
check(old['started_at']!=item['started_at'],'service did not restart: '+item['service'])
check(old['volumes']==item['volumes'],'volume identities changed')
(ROOT/'evidence/persistence.txt').write_text(result.stdout)
save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()})
def public_failover():
cluster.wait_cluster()
try:
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'],
cwd=ROOT,check=True,capture_output=True,text=True,timeout=100)
(ROOT/'evidence/public-failover.txt').write_text(result.stdout)
save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(),
'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())})
finally:
cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90')
cluster.wait_cluster()
def reject_new_connection(survivor, address):
code = '''import json,socket,sys
try:
s=socket.create_connection((sys.argv[1],8080),timeout=1)
except OSError as exc:
print(json.dumps({'accepted':False,'error_type':type(exc).__name__}))
else:
s.close()
print(json.dumps({'accepted':True}))
'''
return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address]))
def failover_each():
cluster.wait_cluster()
originals = cluster.replicas()
check(len(originals)==2,'two running replicas required')
traffic = Traffic().start()
phases = []
try:
time.sleep(3)
baseline_end = time.monotonic()
baseline = traffic.summary(end=baseline_end)
for original in originals:
# Alias changes on restart, so resolve the current process of this exact container.
current = next(r for r in cluster.replicas() if r['container']==original['container'])
all_before = cluster.inspect([current['container']])[0]
address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front'))
survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container'])
read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1'
marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0]
start = time.monotonic()
try:
env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']}
subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10)
stop_returned = time.monotonic()
stopped = json.loads((ROOT/'evidence/last-stop.json').read_text())
read_deadline = start+10
while True:
try:
read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')
check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage')
check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process')
read_done = time.monotonic()
break
except Failure:
check(time.monotonic()<read_deadline,'reads did not recover within10s')
time.sleep(0.1)
check(read_done-start<=10,'read recovery exceeded10s')
rejected = reject_new_connection(survivor,address)
check(not rejected['accepted'],'new connection accepted by stopped API')
time.sleep(max(0,8-(time.monotonic()-start)))
end = time.monotonic()
summary = traffic.summary(start,end)
with traffic.lock:
successful = [a for a in traffic.attempts if a['ok'] and a['started']>=start]
recent = [a for a in successful if a['started']>stop_returned]
recovery = min(a['ended'] for a in recent)-start if recent else None
after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None
check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start')
check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests')
phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4),
'post_stop_write_seconds_from_stop_completion':round(after_stop,4),
'read_during_outage':{'status':200,'confirmed_message_preserved':True,
'seconds_from_action_start':round(read_done-start,4),
'seconds_from_stop_completion':round(read_done-stop_returned,4)},
'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']})
finally:
cluster.command(['docker','start',current['container']])
cluster.wait_healthy(current['container'])
cluster.wait_cluster()
time.sleep(2)
finally:
traffic.stop()
save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases,
'total':traffic.summary(),'data':traffic.verify_data(),
'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2})
def rolling_update():
cluster.wait_cluster()
before = cluster.replicas()
check(len(before)==2,'rolling demonstration expects two replicas')
lb_before = next(item for item in cluster.snapshot() if item['service']=='lb')
target_release = 'lab4-'+uuid.uuid4().hex[:8]
traffic = Traffic().start()
replacements = []
try:
time.sleep(3)
build_start = time.monotonic()
cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180)
build_end = time.monotonic()
migration_output = cluster.compose('run','--rm','--no-deps','migrate')
swap_start = time.monotonic()
for old in before:
step_start = time.monotonic()
result = cluster.replace_one(old['container'])
result['seconds'] = round(time.monotonic()-step_start,4)
replacements.append(result)
swap_end = time.monotonic()
time.sleep(3)
finally:
traffic.stop()
after = cluster.replicas()
lb_after = next(item for item in cluster.snapshot() if item['service']=='lb')
check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test')
check(all(r['release']==target_release for r in after),'new release not observed on all replicas')
check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement')
check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change')
save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release,
'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True,
'build_seconds':round(build_end-build_start,4),'migration_output':migration_output,
'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end),
'total':traffic.summary(),'data':traffic.verify_data(),
'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'})
def database_fixture(container, data):
code = '''import json,sys,psycopg
from psycopg import sql
from messenger.db import setting
p=json.load(sys.stdin)
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c:
if p['action']=='install':
c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text'])))
c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
result={'installed':True}
elif p['action']=='remove':
c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger'])))
c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
result={'removed':True}
else:
result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]}
print(json.dumps(result))
'''
response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT,
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8)
return json.loads(response.stdout)
def graceful_shutdown():
cluster.wait_cluster()
nodes = cluster.replicas()
target,survivor = nodes[0],nodes[1]
address = cluster.front_address(target['container'])
c = Client()
alice,bob,_,chat = c.bootstrap()
path = '/api/v1/chats/'+chat['id']+'/messages'
marker = 'drain_'+uuid.uuid4().hex
fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker}
database_fixture(survivor['container'],{**fixture,'action':'install'})
code = '''import json,sys,urllib.request,urllib.error
p=json.load(sys.stdin)
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']})
try:
response=urllib.request.urlopen(r,timeout=6)
except urllib.error.HTTPError as exc:
response=exc
print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)}))
'''
request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex}
def in_flight():
result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,
input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8)
return json.loads(result.stdout)
pool = ThreadPoolExecutor(max_workers=1)
future = pool.submit(in_flight)
try:
deadline = time.monotonic()+3
while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']:
check(time.monotonic()<deadline and not future.done(),'did not observe an in-flight SQL request')
time.sleep(0.05)
signaled = time.monotonic()
cluster.command(['docker','kill','--signal','SIGTERM',target['container']])
deadline = signaled+1.5
while True:
rejected = reject_new_connection(survivor['container'],address)
still_running = cluster.inspect([target['container']])[0]['State']['Running']
if not rejected['accepted']:
break
check(time.monotonic()<deadline,'new TCP connections still accepted after SIGTERM')
time.sleep(0.05)
check(still_running,'connection refusal observed only after process exit, not during drain')
drained = future.result(timeout=8)
check(drained['status']==201 and drained['instance']==target['instance'],'in-flight request did not finish on signaled process')
check(drained['message']['text']==marker,'in-flight body changed')
exit_code = cluster.command(['docker','wait',target['container']],timeout=8)
elapsed = time.monotonic()-signaled
check(exit_code=='0' and elapsed<5,'SIGTERM exceeded graceful budget or exited with error')
check(history(c,bob,chat)==[drained['message']],'drained message missing or duplicated')
result = {'result':'PASS','target_container':target['container'],'target_instance':target['instance'],
'fixture':'temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards',
'in_flight_observed_in_pg_stat_activity':True,'in_flight_response':201,
'new_tcp_after_sigterm':rejected,'target_still_running_when_new_tcp_rejected':still_running,
'exit_code':int(exit_code),'shutdown_seconds_from_sigterm':round(elapsed,4),
'confirmed_message_preserved':True}
finally:
pool.shutdown(wait=True)
database_fixture(survivor['container'],{**fixture,'action':'remove'})
cluster.command(['docker','start',target['container']])
cluster.wait_healthy(target['container'])
cluster.wait_cluster()
save('graceful-shutdown.json',result)
def infrastructure():
state = cluster.snapshot()
for item in state:
check(item['memory_bytes']>0 and item['nano_cpus']>0,'resource limit missing')
if item['service']!='lb':
check(not item['published_ports'],'service exposed a port bypassing LB')
check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas')
check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running')
save('infrastructure.json',{'result':'PASS','containers':state})
def dependency_failures():
c = Client(timeout=6)
actor = c.user('dependencies')
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
result = {'result':'PASS'}
try:
cluster.compose('stop','-t','2','redis')
response = c.call('GET','/api/v1/users/me',actor=actor,schema='User')
check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed')
c.call('GET','/api/v1/users/me',401)
c.call('GET','/health/ready',schema='Health')
result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200}
finally:
cluster.compose('start','redis')
import urllib.request
import urllib.error
def status(path, token=None):
headers = {'Authorization':'Bearer '+token} if token else {}
request = urllib.request.Request(c.base+path,headers=headers)
started = time.monotonic()
try:
response = urllib.request.urlopen(request,timeout=6)
except urllib.error.HTTPError as error:
response = error
with response:
return response.code, round(time.monotonic()-started,4)
try:
cluster.compose('stop','-t','2','postgres')
code,elapsed = status('/api/v1/users/me',actor['token'])
check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request')
anonymous,_ = status('/api/v1/users/me')
check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage')
live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()]
check(all(code==200 for code in live),'API process liveness depends on DB')
result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous,
'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'}
finally:
cluster.compose('start','postgres')
cluster.wait_cluster()
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
result['same_session_after_recovery'] = True
save('dependency-failures.json',result)
def verify_nginx():
state = cluster.snapshot()
lb = next(item for item in state if item['service']=='lb')
check(lb['image'].startswith('nginx:'),'load balancer must use Nginx')
cluster.command(['docker','exec',lb['container'],'nginx','-t'])
version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1'])
c = Client()
response = c.call('GET','/health/ready',schema='Health')
check(response['headers'].get('Server')=='nginx','public entry point is not Nginx')
check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas')
save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx',
'config_syntax':'PASS','observed_instances':cluster.wait_cluster()})
def main():
parser = argparse.ArgumentParser()
parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?')
args = parser.parse_args()
modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover,
'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure}
if args.mode=='all':
for function in modes.values():
function()
else:
modes[args.mode]()
if __name__ == '__main__':
main()
View File
+297
View File
@@ -0,0 +1,297 @@
"""Contract-compatible synchronous REST API, with short bounded I/O in worker threads."""
import json
import asyncio
import logging
import os
import re
from contextlib import asynccontextmanager
from uuid import uuid4
import psycopg
from fastapi import Depends, FastAPI, Request, Response
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from psycopg_pool import PoolTimeout, TooManyRequests
from redis.exceptions import RedisError
from starlette.exceptions import HTTPException
from starlette.concurrency import run_in_threadpool
from .db import Database
from .errors import ApiError, invalid, not_found, unauthorized
from .models import CreateChat, CreateMessage, CreateUser, uuid_value
from .messages import store_message, validate_key, MESSAGE_FIELDS
from .object_store import ObjectStore, unavailable
from . import attachments
from urllib.parse import quote
from .pagination import decode_cursor, page, page_limit, public, stamp
from .security import HASHER, DUMMY_HASH, basic_credentials, bearer, check_password, digest, new_session
LOG = logging.getLogger('messenger')
USER_FIELDS = 'id, username, display_name, created_at'
def error_response(request, error):
return JSONResponse({'error': {'code': error.code, 'message': error.message,
'request_id': request.state.request_id}},
status_code=error.status, headers=error.headers)
def authenticate(request: Request):
token = bearer(request.headers.get('Authorization', ''))
token_hash = digest(token)
with request.app.state.db.pool.connection() as conn:
row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()',
(token_hash,)).fetchone()
if not row:
raise unauthorized()
request.state.token_hash = token_hash
return row['user_id']
def require_member(conn, chat_id, user_id):
found = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone()
if not found:
raise not_found()
def chat_public(conn, row):
item = public(row)
item['member_ids'] = [str(member['user_id']) for member in conn.execute(
'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()]
return item
def create_app():
@asynccontextmanager
async def lifespan(app):
app.state.db = Database()
app.state.storage = ObjectStore()
app.state.upload_slots = asyncio.Semaphore(6)
app.state.db.open()
try:
yield
finally:
app.state.storage.close()
app.state.db.close()
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
app.state.instance_id = uuid4().hex
app.state.release = os.environ.get('APP_RELEASE', 'lab4-v1')
@app.middleware('http')
async def request_context(request: Request, call_next):
request.state.request_id = str(uuid4())
try:
path = request.url.path
json_endpoint = path in ('/api/v1/users', '/api/v1/chats') or re.fullmatch(r'/api/v1/chats/[^/]+/messages', path)
if request.method == 'POST' and json_endpoint:
media = request.headers.get('Content-Type', '').split(';')
unsupported = media[0].strip().lower() != 'application/json'
for parameter in media[1:]:
name, _, value = parameter.strip().partition('=')
if name.lower() == 'charset' and value.strip('"').lower() not in ('utf-8', 'utf8'):
unsupported = True
if unsupported:
if path != '/api/v1/users':
await run_in_threadpool(authenticate, request)
raise ApiError(415, 'unsupported_media_type', 'Требуется application/json в UTF-8')
try:
json.loads((await request.body()).decode('utf-8'))
except (UnicodeError, json.JSONDecodeError):
raise invalid() from None
response = await call_next(request)
except ApiError as exc:
response = error_response(request, exc)
except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests):
response = error_response(request, ApiError(503, 'unavailable', 'Хранилище временно недоступно'))
except Exception as exc:
# Log exception class only: credentials, SQL and message content stay out of logs.
LOG.error('request_failed request_id=%s type=%s', request.state.request_id, type(exc).__name__)
response = error_response(request, ApiError(500, 'internal_error', 'Внутренняя ошибка'))
response.headers['X-Request-Id'] = request.state.request_id
response.headers['X-Instance-Id'] = app.state.instance_id
response.headers['X-App-Release'] = app.state.release
return response
@app.exception_handler(ApiError)
async def api_error(request, exc):
return error_response(request, exc)
@app.exception_handler(RequestValidationError)
async def validation_error(request, exc):
return error_response(request, invalid())
@app.exception_handler(HTTPException)
async def http_error(request, exc):
return error_response(request, not_found() if exc.status_code == 404 else invalid())
@app.get('/health/live')
def live():
return {'status': 'ok'}
@app.get('/health/ready')
def ready(request: Request):
try:
with request.app.state.db.pool.connection() as conn:
conn.execute('SELECT 1 FROM idempotency_records LIMIT 0')
conn.execute('SELECT 1 FROM attachments LIMIT 0')
except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn):
raise ApiError(503, 'unavailable', 'Схема хранилища ещё не готова') from None
request.app.state.storage.check()
return {'status': 'ok'}
@app.post('/api/v1/users', status_code=201)
def register(body: CreateUser, request: Request):
password_hash = HASHER.hash(body.password)
try:
with request.app.state.db.pool.connection() as conn:
row = conn.execute(
f'INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING {USER_FIELDS}',
(uuid4(), body.username, body.display_name, password_hash)).fetchone()
except psycopg.errors.UniqueViolation:
raise ApiError(409, 'conflict', 'Имя пользователя занято') from None
return public(row)
@app.post('/api/v1/auth/sessions', status_code=201)
async def create_session(request: Request):
# Run hashing and synchronous database calls outside the event loop.
username, password = basic_credentials(request.headers.get('Authorization', ''))
if await request.body():
raise invalid()
return await run_in_threadpool(login, request, username, password)
def login(request, username, password):
db = request.app.state.db
with db.pool.connection() as conn:
row = conn.execute('SELECT '+USER_FIELDS+', password_hash FROM users WHERE username=%s', (username,)).fetchone()
valid = check_password(row['password_hash'] if row else DUMMY_HASH, password)
if not row or not valid:
raise unauthorized(basic=True)
row.pop('password_hash')
token, expires_at = new_session(db.session_ttl)
with db.pool.connection() as conn:
conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)',
(digest(token), row['id'], expires_at))
return {'token': token, 'token_type': 'Bearer', 'expires_at': stamp(expires_at), 'user': public(row)}
@app.get('/api/v1/users/me')
def me(request: Request, user_id=Depends(authenticate)):
db = request.app.state.db
key = 'user:' + str(user_id)
try:
cached = db.cache.get(key)
if cached:
return json.loads(cached)
except (RedisError, ValueError):
pass
with db.pool.connection() as conn:
row = conn.execute('SELECT '+USER_FIELDS+' FROM users WHERE id=%s', (user_id,)).fetchone()
if not row:
raise unauthorized()
result = public(row)
try:
db.cache.set(key, json.dumps(result), ex=60)
except RedisError:
pass
return result
@app.delete('/api/v1/auth/sessions/current', status_code=204)
def logout(request: Request, user_id=Depends(authenticate)):
with request.app.state.db.pool.connection() as conn:
conn.execute('DELETE FROM sessions WHERE token_hash=%s', (request.state.token_hash,))
return Response(status_code=204)
@app.post('/api/v1/chats', status_code=201)
def create_chat(body: CreateChat, request: Request, user_id=Depends(authenticate)):
if str(user_id) not in body.member_ids:
raise invalid()
try:
with request.app.state.db.pool.connection() as conn:
row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at',
(uuid4(), body.title)).fetchone()
# An unknown user raises AFTER the chat INSERT; transaction context rolls it all back.
with conn.cursor() as cur:
cur.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)',
[(row['id'], uuid_value(member)) for member in body.member_ids])
result = chat_public(conn, row)
except psycopg.errors.ForeignKeyViolation:
raise invalid() from None
return result
@app.get('/api/v1/chats')
def list_chats(request: Request, user_id=Depends(authenticate)):
limit = page_limit(request.query_params.get('limit', '50'))
scope = 'chats:' + str(user_id)
after = decode_cursor(scope, request.query_params.get('cursor'))
where = ' AND (c.created_at,c.id) > (%s,%s)' if after else ''
params = (user_id, *after, limit+1) if after else (user_id, limit+1)
with request.app.state.db.pool.connection() as conn:
rows = conn.execute('SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id '
'WHERE m.user_id=%s'+where+' ORDER BY c.created_at,c.id LIMIT %s', params).fetchall()
result = page(scope, rows, limit)
result['items'] = [chat_public(conn, row) for row in rows[:limit]]
return result
@app.get('/api/v1/chats/{chat_id}')
def get_chat(chat_id: str, request: Request, user_id=Depends(authenticate)):
cid = uuid_value(chat_id)
with request.app.state.db.pool.connection() as conn:
require_member(conn, cid, user_id)
row = conn.execute('SELECT id,title,created_at FROM chats WHERE id=%s', (cid,)).fetchone()
return chat_public(conn, row)
@app.post('/api/v1/chats/{chat_id}/messages', status_code=201)
def send_message(chat_id: str, body: CreateMessage, request: Request, user_id=Depends(authenticate)):
cid = uuid_value(chat_id)
key = validate_key(request.headers.get('Idempotency-Key', ''))
with request.app.state.db.pool.connection() as conn:
require_member(conn, cid, user_id)
row = store_message(conn, user_id, cid, body.text, key, body.attachment_ids)
return public(row)
@app.get('/api/v1/chats/{chat_id}/messages')
def history(chat_id: str, request: Request, user_id=Depends(authenticate)):
cid = uuid_value(chat_id)
limit = page_limit(request.query_params.get('limit', '50'))
scope = 'messages:' + str(cid)
with request.app.state.db.pool.connection() as conn:
require_member(conn, cid, user_id)
after = decode_cursor(scope, request.query_params.get('cursor'))
where = ' AND (m.created_at,m.id) > (%s,%s)' if after else ''
params = (cid, *after, limit+1) if after else (cid, limit+1)
rows = conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.chat_id=%s'+where+
' ORDER BY m.created_at,m.id LIMIT %s', params).fetchall()
return page(scope, rows, limit)
@app.post('/api/v1/chats/{chat_id}/attachments',status_code=201)
async def upload_attachment(chat_id: str,request: Request,filename: str='',user_id=Depends(authenticate)):
cid=uuid_value(chat_id)
def access():
with request.app.state.db.pool.connection() as conn:
require_member(conn,cid,user_id)
await run_in_threadpool(access)
return await attachments.upload(request,user_id,cid,filename)
@app.get('/api/v1/attachments/{attachment_id}')
def attachment_metadata(attachment_id: str,request: Request,user_id=Depends(authenticate)):
aid=uuid_value(attachment_id)
with request.app.state.db.pool.connection() as conn:
row=attachments.lookup(conn,aid,user_id)
return attachments.metadata(row)
@app.get('/api/v1/attachments/{attachment_id}/content')
def attachment_content(attachment_id: str,request: Request,variant: str='original',user_id=Depends(authenticate)):
aid=uuid_value(attachment_id)
with request.app.state.db.pool.connection() as conn:
row=attachments.lookup(conn,aid,user_id)
if variant!='original':
raise invalid()
opened=request.app.state.storage.get(row['object_key'])
if opened.length!=row['size_bytes']:
opened.close()
raise unavailable()
return attachments.ObjectResponse(opened,media_type=row['content_type'],headers={
'Content-Length':str(row['size_bytes']),
'Content-Disposition':"inline; filename*=UTF-8''"+quote(row['filename'],safe='')})
return app
+176
View File
@@ -0,0 +1,176 @@
"""Bounded binary upload, image validation and DB/S3 coordination."""
import asyncio
import hashlib
import logging
import tempfile
import threading
import unicodedata
import warnings
from uuid import uuid4
import psycopg
from PIL import Image, PngImagePlugin, UnidentifiedImageError
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
from starlette.responses import StreamingResponse
from .errors import ApiError, invalid, not_found
from .pagination import public
MAX_BYTES=10485760
MAX_PIXELS=40000000
Image.MAX_IMAGE_PIXELS=MAX_PIXELS
PngImagePlugin.MAX_TEXT_CHUNK=1048576
PngImagePlugin.MAX_TEXT_MEMORY=1048576
warnings.filterwarnings('error',category=Image.DecompressionBombWarning)
DECODER=threading.BoundedSemaphore(1)
LOG=logging.getLogger('messenger')
FIELDS='id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at'
def object_lock(conn,key,wait=True):
function='pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock'
row=conn.execute('SELECT '+function+'(hashtextextended(%s,0)) AS locked',(key,)).fetchone()
return True if wait else row['locked']
def filename_value(value):
if not value or not value.strip() or len(value)>255:
raise invalid()
try:
value.encode('utf-8')
except UnicodeError:
raise invalid() from None
if any(c in '/\\' or unicodedata.category(c)=='Cc' for c in value):
raise invalid()
return value
def metadata(row):
result=public({key:value for key,value in row.items() if key!='object_key'})
result.update(status='ready',variants=[],error_code=None)
return result
def validate_image(file,mime):
if not DECODER.acquire(timeout=3):
raise ApiError(503,'unavailable','Обработка изображений занята')
try:
file.seek(0)
with Image.open(file) as image:
actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(image.format)
if actual is None or actual!=mime:
raise ApiError(415,'unsupported_media_type','Ожидается PNG или JPEG с правильным MIME')
if image.width*image.height>MAX_PIXELS:
raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей')
image.verify()
file.seek(0)
with Image.open(file) as image:
image.load()
file.seek(0)
except (Image.DecompressionBombWarning,Image.DecompressionBombError):
raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей') from None
except (UnidentifiedImageError,OSError,SyntaxError,ValueError,EOFError):
raise ApiError(415,'unsupported_media_type','Повреждённое или неподдерживаемое изображение') from None
finally:
DECODER.release()
def compensate(db,store,key):
"""A failed/unknown COMMIT is not proof of absence. Recheck under the same lock."""
try:
with db.pool.connection() as conn:
if not object_lock(conn,key,wait=False):
return
exists=conn.execute('SELECT 1 FROM attachments WHERE object_key=%s',(key,)).fetchone()
if not exists:
store.delete(key)
except Exception as exc:
# A periodic, grace-protected janitor will retry after dependency recovery.
LOG.warning('attachment_compensation_deferred type=%s',type(exc).__name__)
def persist(db,store,user_id,chat_id,filename,mime,file,length,digest):
key='originals/'+uuid4().hex
try:
with db.pool.connection() as conn:
object_lock(conn,key)
store.put(key,file,length,mime)
row=conn.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key) '
'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING '+FIELDS,
(uuid4(),chat_id,user_id,filename,mime,length,digest,key)).fetchone()
return metadata(row)
except ApiError:
compensate(db,store,key)
raise
except psycopg.Error:
compensate(db,store,key)
raise ApiError(503,'unavailable','Не удалось сохранить вложение') from None
except BaseException:
compensate(db,store,key)
raise
async def upload(request,user_id,chat_id,filename):
filename=filename_value(filename)
mime=request.headers.get('Content-Type','').split(';')[0].strip().lower()
if mime not in ('image/png','image/jpeg'):
raise ApiError(415,'unsupported_media_type','Ожидается image/png или image/jpeg')
declared=request.headers.get('Content-Length')
if declared is not None:
try:
length=int(declared)
if length<0:
raise ValueError
except ValueError:
raise invalid() from None
if length>MAX_BYTES:
raise ApiError(413,'payload_too_large','Файл превышает 10 MiB')
try:
await asyncio.wait_for(request.app.state.upload_slots.acquire(),timeout=1)
except TimeoutError:
raise ApiError(503,'unavailable','Слишком много загрузок') from None
try:
with tempfile.TemporaryFile(mode='w+b') as file:
total=0
digest=hashlib.sha256()
async def read_body():
nonlocal total
async for chunk in request.stream():
total+=len(chunk)
if total>MAX_BYTES:
raise ApiError(413,'payload_too_large','Файл превышает 10 MiB')
digest.update(chunk)
await run_in_threadpool(file.write,chunk)
try:
await asyncio.wait_for(read_body(),timeout=30)
except TimeoutError:
raise ApiError(400,'invalid_request','Время загрузки истекло') from None
except ClientDisconnect:
raise invalid() from None
await run_in_threadpool(validate_image,file,mime)
return await run_in_threadpool(persist,request.app.state.db,request.app.state.storage,
user_id,chat_id,filename,mime,file,total,digest.hexdigest())
finally:
request.app.state.upload_slots.release()
def lookup(conn,attachment_id,user_id):
row=conn.execute('SELECT '+', '.join('a.'+field for field in (FIELDS+',object_key').split(','))+
' FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id '
'WHERE a.id=%s AND m.user_id=%s',(attachment_id,user_id)).fetchone()
if not row:
raise not_found()
return row
class ObjectResponse(StreamingResponse):
def __init__(self,opened,**kwargs):
self.opened=opened
super().__init__(opened.chunks(),**kwargs)
async def __call__(self,scope,receive,send):
try:
await super().__call__(scope,receive,send)
finally:
self.opened.close()
+72
View File
@@ -0,0 +1,72 @@
"""Grace-protected orphan cleanup; absence is verified in the primary DB under lock."""
import argparse
from datetime import datetime,timezone,timedelta
import json
from pathlib import Path
import time
from .attachments import object_lock
from .db import Database
from .object_store import ObjectStore
def sweep(db,store,grace_seconds=300,apply=False):
if grace_seconds<60:
raise ValueError('Grace must be at least60seconds')
cutoff=datetime.now(timezone.utc)-timedelta(seconds=grace_seconds)
result={'mode':'apply' if apply else 'dry-run','grace_seconds':grace_seconds,
'listed':0,'fresh':0,'tracked':0,'locked':0,'orphans':0,'deleted':0}
for page in store.pages():
for obj in page.get('Contents',[]):
result['listed']+=1
if obj['LastModified']>=cutoff:
result['fresh']+=1
continue
with db.pool.connection() as conn:
if not object_lock(conn,obj['Key'],wait=False):
result['locked']+=1
continue
if conn.execute('SELECT 1 FROM attachments WHERE object_key=%s',(obj['Key'],)).fetchone():
result['tracked']+=1
continue
result['orphans']+=1
if apply:
store.delete(obj['Key'])
result['deleted']+=1
return result
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--grace-seconds',type=int,default=300)
parser.add_argument('--apply',action='store_true')
parser.add_argument('--dry-run',action='store_true')
parser.add_argument('--loop',action='store_true')
parser.add_argument('--interval',type=int,default=60)
args=parser.parse_args()
if args.grace_seconds<60 or args.interval<1 or (args.apply and args.dry_run):
parser.error('Use grace>=60, interval>=1 and either --apply or --dry-run')
db=Database()
store=ObjectStore()
db.open()
try:
while True:
try:
result=sweep(db,store,args.grace_seconds,args.apply)
print(json.dumps(result),flush=True)
except Exception as exc:
print(json.dumps({'error':type(exc).__name__,'deleted_unknown':False}),flush=True)
if not args.loop:
raise
finally:
Path('/tmp/cleanup-heartbeat').touch()
if not args.loop:
break
time.sleep(args.interval)
finally:
store.close()
db.close()
if __name__=='__main__':
main()
+42
View File
@@ -0,0 +1,42 @@
"""Bounded dependencies; PostgreSQL is always the source of truth."""
import os
from pathlib import Path
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from redis import Redis
from redis.backoff import NoBackoff
from redis.retry import Retry
def setting(name, default=None):
filename = os.environ.get(name + '_FILE')
value = Path(filename).read_text().strip() if filename else os.environ.get(name, default)
if value is None:
raise RuntimeError(f'Missing configuration: {name}')
return value
class Database:
def __init__(self):
self.session_ttl = int(setting('SESSION_TTL_SECONDS', '3600'))
if self.session_ttl < 1:
raise ValueError('SESSION_TTL_SECONDS must be positive')
self.pool = ConnectionPool(
setting('DATABASE_URL'), min_size=1, max_size=int(setting('POOL_MAX_SIZE', '8')),
timeout=2, max_waiting=16, open=False,
kwargs={'row_factory': dict_row, 'connect_timeout': 3,
'options': '-c statement_timeout=3000 -c lock_timeout=2000 -c idle_in_transaction_session_timeout=5000'},
)
self.cache = Redis.from_url(
setting('CACHE_URL', 'redis://redis:6379/0'), decode_responses=True,
socket_connect_timeout=0.2, socket_timeout=0.2, max_connections=8,
retry=Retry(NoBackoff(), 0),
)
def open(self):
self.pool.open(wait=True, timeout=30)
def close(self):
self.pool.close()
self.cache.close()
+19
View File
@@ -0,0 +1,19 @@
class ApiError(Exception):
def __init__(self, status, code, message, headers=None):
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def invalid():
return ApiError(400, 'invalid_request', 'Неверный запрос')
def unauthorized(basic=False):
scheme = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"'
return ApiError(401, 'unauthorized', 'Требуется аутентификация', {'WWW-Authenticate': scheme})
def not_found():
return ApiError(404, 'not_found', 'Объект не найден')
+52
View File
@@ -0,0 +1,52 @@
"""One PostgreSQL transaction owns both a message and its retry result."""
import hashlib
import json
import re
from uuid import uuid4
from .errors import ApiError, invalid, not_found
from .models import uuid_value
def validate_key(key):
if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
raise invalid()
return key
MESSAGE_FIELDS = 'm.id,m.chat_id,m.sender_id,m.text,m.created_at,ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY position) AS attachment_ids'
def store_message(conn, user_id, chat_id, text, key, attachment_ids=None):
attachment_ids=[] if attachment_ids is None else attachment_ids
payload={'text':text}
if attachment_ids:
payload['attachment_ids']=attachment_ids
canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
fingerprint = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
params = (user_id, chat_id, key)
# A competing INSERT waits for the other transaction, then the next statement sees its result.
conn.execute("INSERT INTO idempotency_records(user_id,chat_id,key,request_hash,expires_at) "
"VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING",
(*params, fingerprint))
record = conn.execute('SELECT request_hash,message_id,expires_at > clock_timestamp() AS active '
'FROM idempotency_records WHERE user_id=%s AND chat_id=%s AND key=%s FOR UPDATE', params).fetchone()
if record['message_id'] is not None and record['active']:
if record['request_hash'] != fingerprint:
raise ApiError(409, 'conflict', 'Ключ уже использован с другим сообщением')
return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',
(record['message_id'],)).fetchone()
ids=[uuid_value(item) for item in attachment_ids]
if ids:
allowed=conn.execute('SELECT id FROM attachments WHERE chat_id=%s AND id=ANY(%s)',(chat_id,ids)).fetchall()
if len(allowed)!=len(ids):
raise not_found()
row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) '
'RETURNING id,chat_id,sender_id,text,created_at', (uuid4(), chat_id, user_id, text)).fetchone()
if ids:
with conn.cursor() as cur:
cur.executemany('INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)',
[(row['id'],aid,i) for i,aid in enumerate(ids)])
conn.execute("UPDATE idempotency_records SET request_hash=%s,message_id=%s,expires_at=clock_timestamp()+interval '24 hours' "
'WHERE user_id=%s AND chat_id=%s AND key=%s', (fingerprint, row['id'], *params))
return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',(row['id'],)).fetchone()
+29
View File
@@ -0,0 +1,29 @@
"""Versioned, atomic, repeatable migrations, serialized with an advisory lock."""
import hashlib
from pathlib import Path
import psycopg
from .db import setting
def main():
directory = Path(__file__).resolve().parents[2] / 'migrations'
with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn:
conn.execute('SELECT pg_advisory_xact_lock(20261102)')
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())')
for path in sorted(directory.glob('*.sql')):
sql = path.read_text()
checksum = hashlib.sha256(sql.encode()).hexdigest()
found = conn.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone()
if found:
if found[0] != checksum:
raise RuntimeError('Applied migration checksum changed: ' + path.name)
continue
conn.execute(sql)
conn.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (path.name, checksum))
print('Applied', path.name)
print('Migrations up to date')
if __name__ == '__main__':
main()
+75
View File
@@ -0,0 +1,75 @@
import re
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from .errors import invalid
def uuid_value(value):
try:
# Require the UUID string representation, not integer/coercible values.
if not isinstance(value, str) or len(value) != 36:
raise ValueError
return UUID(value)
except (ValueError, TypeError, AttributeError):
raise invalid() from None
class StrictModel(BaseModel):
model_config = ConfigDict(extra='forbid', strict=True)
@field_validator('*')
@classmethod
def valid_strings(cls, value, info):
if isinstance(value, str):
if not value.strip() and info.field_name != 'text':
raise ValueError('Blank string')
# PostgreSQL cannot store NUL or unpaired Unicode surrogates.
if '\x00' in value:
raise ValueError('NUL is not supported')
value.encode('utf-8')
return value
class CreateUser(StrictModel):
username: str = Field(pattern=r'^[a-z0-9_]{3,32}$')
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator('username')
@classmethod
def username_ascii(cls, value):
if not re.fullmatch(r'[a-z0-9_]{3,32}', value):
raise ValueError('Invalid username')
return value
class CreateChat(StrictModel):
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
@field_validator('member_ids')
@classmethod
def members(cls, value):
ids = [uuid_value(item) for item in value]
if len(ids) != len(set(ids)):
raise ValueError('Duplicate member')
return [str(item) for item in ids]
class CreateMessage(StrictModel):
text: str = Field(max_length=4000)
attachment_ids: list[str] = Field(default_factory=list,max_length=10)
@field_validator('attachment_ids')
@classmethod
def attachment_values(cls,value):
ids=[uuid_value(item) for item in value]
if len(ids)!=len(set(ids)):
raise ValueError('Duplicate attachment')
return [str(item) for item in ids]
@model_validator(mode='after')
def nonempty_message(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError('Message must contain text or attachment')
return self
+109
View File
@@ -0,0 +1,109 @@
"""Small bounded S3 client. API credentials never contain root/admin permissions."""
from contextlib import contextmanager
import threading
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from .db import setting
from .errors import ApiError
PREFIX = 'originals/'
def unavailable():
return ApiError(503,'unavailable','Хранилище файлов временно недоступно')
class OpenObject:
def __init__(self,body,release,length):
self.body=body
self.release=release
self.length=length
self.closed=False
self.close_lock=threading.Lock()
def chunks(self):
try:
while True:
data=self.body.read(65536)
if not data:
break
yield data
finally:
self.close()
def close(self):
with self.close_lock:
if self.closed:
return
self.closed=True
try:
self.body.close()
finally:
self.release()
class ObjectStore:
def __init__(self):
self.bucket=setting('S3_BUCKET','messenger-attachments')
self.gate=threading.BoundedSemaphore(8)
self.client=boto3.client('s3',endpoint_url=setting('S3_ENDPOINT','http://minio:9000'),
region_name=setting('S3_REGION','us-east-1'),
aws_access_key_id=setting('S3_ACCESS_KEY_ID'),aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'),
config=Config(signature_version='s3v4',connect_timeout=1,read_timeout=3,max_pool_connections=8,
retries={'total_max_attempts':1,'mode':'standard'},s3={'addressing_style':'path'},
request_checksum_calculation='when_required',response_checksum_validation='when_required'))
@contextmanager
def operation(self):
if not self.gate.acquire(timeout=1):
raise unavailable()
try:
yield
except (BotoCoreError,ClientError):
raise unavailable() from None
finally:
self.gate.release()
def check(self):
with self.operation():
self.client.list_objects_v2(Bucket=self.bucket,Prefix=PREFIX,MaxKeys=1)
def put(self,key,body,length,mime):
with self.operation():
body.seek(0)
self.client.put_object(Bucket=self.bucket,Key=key,Body=body,ContentLength=length,ContentType=mime)
def delete(self,key):
with self.operation():
self.client.delete_object(Bucket=self.bucket,Key=key)
def get(self,key):
if not self.gate.acquire(timeout=1):
raise unavailable()
try:
result=self.client.get_object(Bucket=self.bucket,Key=key)
return OpenObject(result['Body'],self.gate.release,result['ContentLength'])
except (BotoCoreError,ClientError):
self.gate.release()
raise unavailable() from None
except BaseException:
self.gate.release()
raise
def pages(self):
token=None
while True:
params={'Bucket':self.bucket,'Prefix':PREFIX,'MaxKeys':1000}
if token:
params['ContinuationToken']=token
with self.operation():
result=self.client.list_objects_v2(**params)
yield result
if not result.get('IsTruncated'):
break
token=result['NextContinuationToken']
def close(self):
self.client.close()
+56
View File
@@ -0,0 +1,56 @@
import base64
import json
import re
from datetime import datetime, timezone
from uuid import UUID
from .errors import invalid
from .models import uuid_value
def stamp(value):
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
def encode_cursor(scope, row):
raw = json.dumps([scope, stamp(row['created_at']), str(row['id'])], separators=(',', ':')).encode()
return base64.urlsafe_b64encode(raw).decode().rstrip('=')
def decode_cursor(scope, value):
if value is None:
return None
try:
if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', value):
raise ValueError
data = json.loads(base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True))
if not isinstance(data, list) or len(data) != 3 or data[0] != scope:
raise ValueError
ts = datetime.fromisoformat(data[1].replace('Z', '+00:00'))
if ts.tzinfo is None:
raise ValueError
return ts, uuid_value(data[2])
except (ValueError, TypeError, AttributeError, OverflowError):
raise invalid() from None
def page_limit(value):
if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100:
raise invalid()
return int(value)
def public(row):
def value(item):
if isinstance(item,datetime):
return stamp(item)
if isinstance(item,UUID):
return str(item)
if isinstance(item,list):
return [value(part) for part in item]
return item
return {key:value(item) for key,item in row.items()}
def page(scope, rows, limit):
return {'items': [public(row) for row in rows[:limit]],
'next_cursor': encode_cursor(scope, rows[limit-1]) if len(rows) > limit else None}
+49
View File
@@ -0,0 +1,49 @@
import base64
import hashlib
import re
import secrets
from datetime import datetime, timedelta, timezone
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError, InvalidHashError
from .errors import unauthorized
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
DUMMY_HASH = HASHER.hash('synthetic-dummy-password')
def digest(token):
return hashlib.sha256(token.encode()).hexdigest()
def basic_credentials(header):
try:
scheme, encoded = header.split(' ', 1)
if scheme.lower() != 'basic':
raise ValueError
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
if not re.fullmatch(r'[a-z0-9_]{3,32}', username) or not 12 <= len(password) <= 128:
raise ValueError
return username, password
except (ValueError, UnicodeError):
raise unauthorized(basic=True) from None
def check_password(encoded, password):
try:
return HASHER.verify(encoded, password)
except (VerificationError, InvalidHashError):
return False
def bearer(header):
try:
scheme, token = header.split(' ', 1)
if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
raise ValueError
return token
except ValueError:
raise unauthorized() from None
def new_session(ttl):
return secrets.token_urlsafe(32), datetime.now(timezone.utc) + timedelta(seconds=ttl)
+141
View File
@@ -0,0 +1,141 @@
"""Integration tests: real PostgreSQL and Redis, isolated synthetic identities."""
import base64
import json
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
import pytest
from fastapi.testclient import TestClient
from messenger.app import create_app
PASSWORD = 'Тестовый:пароль🔐' * 8
@pytest.fixture
def client():
with TestClient(create_app()) as c:
yield c
def user(c):
body = dict(username='u_' + uuid.uuid4().hex[:24], display_name='Алиса 👋', password=PASSWORD)
r = c.post('/api/v1/users', json=body)
assert r.status_code == 201, r.text
return r.json(), body
def login(c, body):
credentials = base64.b64encode((body['username'] + ':' + body['password']).encode()).decode()
r = c.post('/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credentials})
assert r.status_code == 201, r.text
return {'Authorization': 'Bearer ' + r.json()['token']}, r.json()
def test_auth_unicode_logout_and_cache(client):
u, body = user(client)
h, session = login(client, body)
assert session['user'] == u
assert len(session['token']) >= 43
key = 'user:' + u['id']
client.app.state.db.cache.delete(key)
for _ in range(2):
assert client.get('/api/v1/users/me', headers=h).json() == u
cached = json.loads(client.app.state.db.cache.get(key))
assert cached == u
assert 0 < client.app.state.db.cache.ttl(key) <= 60
assert client.delete('/api/v1/auth/sessions/current', headers=h).status_code == 204
r = client.get('/api/v1/users/me', headers=h)
assert r.status_code == 401
assert r.headers['www-authenticate'] == 'Bearer realm="messenger"'
assert r.json()['error']['request_id'] == r.headers['x-request-id']
def test_session_expiry(monkeypatch):
monkeypatch.setenv('SESSION_TTL_SECONDS', '2')
with TestClient(create_app()) as client:
_, body = user(client)
h, _ = login(client, body)
assert client.get('/api/v1/users/me', headers=h).status_code == 200
time.sleep(2.1)
assert client.get('/api/v1/users/me', headers=h).status_code == 401
def test_registration_race(client):
body = dict(username='race_' + uuid.uuid4().hex[:22], display_name='Race', password=PASSWORD)
with ThreadPoolExecutor(max_workers=6) as executor:
codes = list(executor.map(lambda _: client.post('/api/v1/users', json=body).status_code, range(6)))
assert sorted(codes) == [201, 409, 409, 409, 409, 409]
def test_chat_rollback_after_insert(client):
u, body = user(client)
h, _ = login(client, body)
title = 'rollback_' + uuid.uuid4().hex
r = client.post('/api/v1/chats', headers=h, json={'title': title, 'member_ids': [u['id'], str(uuid.uuid4())]})
assert r.status_code == 400
with client.app.state.db.pool.connection() as conn:
assert conn.execute('SELECT count(*) AS n FROM chats WHERE title=%s', (title,)).fetchone()['n'] == 0
assert conn.execute('SELECT count(*) AS n FROM chat_members m LEFT JOIN chats c ON c.id=m.chat_id WHERE c.id IS NULL').fetchone()['n'] == 0
def test_cursor_access_and_strict_validation(client):
a, ab = user(client)
b, bb = user(client)
ah, _ = login(client, ab)
bh, _ = login(client, bb)
chats = [client.post('/api/v1/chats', headers=ah, json={'title':'Чат','member_ids':[a['id']]}).json() for _ in range(2)]
path = '/api/v1/chats/' + chats[0]['id']
for text in ['Один 👋', 'Два', 'Три']:
assert client.post(path+'/messages', headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, json={'text':text}).status_code == 201
p = client.get(path+'/messages?limit=2', headers=ah).json()
assert [m['text'] for m in p['items']] == ['Один 👋','Два']
tail = client.get(path+'/messages', headers=ah, params={'cursor':p['next_cursor'],'limit':1}).json()
assert [m['text'] for m in tail['items']] == ['Три']
assert tail['next_cursor'] is None
r = client.get('/api/v1/chats/'+chats[1]['id']+'/messages',headers=ah, params={'cursor':p['next_cursor']})
assert r.status_code == 400
assert client.get(path, headers=bh).status_code == 404
for payload in [{'text':123}, {'text':' '}, {'text':'ok','sender_id':b['id']}, {'text':'x'*4001}]:
assert client.post(path+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json=payload).status_code == 400
for query in ['limit=0','limit=101','limit=1.0','cursor=bad','cursor=']:
assert client.get(path+'/messages?'+query,headers=ah).status_code == 400
for payload in [{'username':123,'display_name':'a','password':PASSWORD}, {'username':'valid','display_name':' ','password':PASSWORD}]:
assert client.post('/api/v1/users',json=payload).status_code == 400
assert client.post('/api/v1/users',content='{',headers={'Content-Type':'application/json'}).status_code == 400
def test_session_storage_and_migrations(client):
from messenger.migrate import main as migrate
u, body = user(client)
_, session = login(client, body)
db = client.app.state.db
def snapshot():
with db.pool.connection() as conn:
return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n']
for table in ['users','chats','chat_members','messages','sessions','schema_migrations','idempotency_records']}
before = snapshot()
migrate()
migrate()
assert snapshot() == before
with db.pool.connection() as conn:
row = conn.execute('SELECT token_hash,expires_at FROM sessions WHERE user_id=%s', (u['id'],)).fetchone()
assert row['token_hash'] != session['token']
assert len(row['token_hash']) == 64
stored = conn.execute('SELECT password_hash FROM users WHERE id=%s', (u['id'],)).fetchone()['password_hash']
assert stored.startswith('$argon2id$')
assert stored != body['password']
def test_json_media_type_and_utf8(client):
body = dict(username='media_' + uuid.uuid4().hex[:24], display_name='UTF8', password=PASSWORD)
raw = json.dumps(body).encode()
for headers in [{}, {'Content-Type':'text/plain'}, {'Content-Type':'application/xml'}]:
r = client.post('/api/v1/users',content=raw,headers=headers)
assert r.status_code == 415
assert r.json()['error']['code'] == 'unsupported_media_type'
for encoding in ['utf-16', 'utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be']:
r = client.post('/api/v1/users',content=json.dumps(body).encode(encoding),headers={'Content-Type':'application/json'})
assert r.status_code == 400, encoding
r = client.post('/api/v1/chats',content=b'{}',headers={'Content-Type':'text/plain'})
assert r.status_code == 401
+167
View File
@@ -0,0 +1,167 @@
"""Real PostgreSQL + private S3 integration; synthetic images only."""
import hashlib
import io
import struct
import uuid
import zlib
from PIL import Image
from psycopg import sql
from test_api import client, user, login
def image_bytes(kind='PNG'):
stream = io.BytesIO()
Image.new('RGB',(320,200),(45,110,190)).save(stream,format=kind)
return stream.getvalue()
def actors(c):
a,ab=user(c)
b,bb=user(c)
e,eb=user(c)
ah,_=login(c,ab)
bh,_=login(c,bb)
eh,_=login(c,eb)
chat=c.post('/api/v1/chats',headers=ah,json={'title':'Файлы','member_ids':[a['id'],b['id']]}).json()
return ah,bh,eh,chat
def upload(c,headers,chat,raw=None,mime='image/png',filename='Фото.png'):
raw=image_bytes() if raw is None else raw
return c.post('/api/v1/chats/'+chat['id']+'/attachments',params={'filename':filename},content=raw,
headers={**headers,'Content-Type':mime})
def test_original_png_jpeg_and_message(client):
ah,bh,eh,chat=actors(client)
for kind,mime in [('PNG','image/png'),('JPEG','image/jpeg')]:
raw=image_bytes(kind)
r=upload(client,ah,chat,raw,mime)
assert r.status_code==201
att=r.json()
assert att['status']=='ready' and att['variants']==[] and att['error_code'] is None
assert att['sha256']==hashlib.sha256(raw).hexdigest() and att['size_bytes']==len(raw)
assert 'object_key' not in att and 'bucket' not in att
path='/api/v1/attachments/'+att['id']
assert client.get(path,headers=bh).json()==att
content=client.get(path+'/content',headers=bh)
assert content.status_code==200 and content.content==raw
assert content.headers['Content-Type']==mime
for suffix in ['', '/content']:
assert client.get(path+suffix,headers=eh).status_code==404
assert client.get(path+'/content?variant=thumbnail',headers=bh).status_code==400
response=client.post('/api/v1/chats/'+chat['id']+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},
json={'text':'','attachment_ids':[att['id']]})
assert response.status_code==201
assert response.json()['attachment_ids']==[att['id']]
def test_invalid_media_corruption_limits_and_filename(client):
ah,_,_,chat=actors(client)
cases=[(b'not an image','image/png',415),(image_bytes(),'image/jpeg',415),
(image_bytes('JPEG'),'image/png',415),(image_bytes()[:40],'image/png',415),
(image_bytes(),'text/plain',415),(b'x'*(10485760+1),'image/png',413)]
for raw,mime,expected in cases:
response=upload(client,ah,chat,raw,mime)
assert response.status_code==expected
assert response.json()['error']['request_id']==response.headers['X-Request-Id']
# Valid PNG header advertises >40Mp; reject before attempting decompression.
header=struct.pack('!IIBBBBB',10001,4000,8,2,0,0,0)
chunk=lambda tag,data: struct.pack('!I',len(data))+tag+data+struct.pack('!I',zlib.crc32(tag+data)&0xffffffff)
bomb=b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',header)+chunk(b'IDAT',zlib.compress(b''))+chunk(b'IEND',b'')
assert upload(client,ah,chat,bomb).status_code==413
for filename in ['../x.png','a\\b.png','bad\x00.png','bad\x1f.png','bad\x7f.png',' ','x'*256]:
assert upload(client,ah,chat,filename=filename).status_code==400
def test_foreign_attachment_binding_and_normalized_replay(client):
ah,bh,eh,chat=actors(client)
att=upload(client,ah,chat).json()
other=client.post('/api/v1/chats',headers=bh,json={'title':'Другой чат','member_ids':[chat['member_ids'][0],chat['member_ids'][1]]}).json()
path='/api/v1/chats/'+other['id']+'/messages'
for aid in [att['id'],str(uuid.uuid4())]:
response=client.post(path,headers={**bh,'Idempotency-Key':uuid.uuid4().hex},json={'text':'test','attachment_ids':[aid]})
assert response.status_code==404
assert upload(client,eh,chat).status_code==404
h={**ah,'Idempotency-Key':uuid.uuid4().hex}
path='/api/v1/chats/'+chat['id']+'/messages'
first=client.post(path,headers=h,json={'text':'same'})
replay=client.post(path,headers=h,json={'attachment_ids':[],'text':'same'})
assert first.status_code==replay.status_code==201 and first.json()==replay.json()
conflict=client.post(path,headers=h,json={'text':'same','attachment_ids':[att['id']]})
assert conflict.status_code==409
duplicates=client.post(path,headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[att['id'],att['id']]})
assert duplicates.status_code==400
def test_compensate_s3_when_metadata_insert_fails(client):
ah,_,_,chat=actors(client)
db=client.app.state.db
store=client.app.state.storage
before={item['Key'] for page in store.pages() for item in page.get('Contents',[])}
function='fail_attachment_'+uuid.uuid4().hex
trigger='fail_attachment_'+uuid.uuid4().hex
filename='fault_'+uuid.uuid4().hex+'.png'
with db.pool.connection() as conn:
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename={} THEN RAISE EXCEPTION 'synthetic metadata failure'; END IF; RETURN NEW; END $$").format(sql.Identifier(function),sql.Literal(filename)))
conn.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
try:
response=upload(client,ah,chat,filename=filename)
assert response.status_code==503
with db.pool.connection() as conn:
assert conn.execute('SELECT count(*) AS n FROM attachments WHERE filename=%s',(filename,)).fetchone()['n']==0
after={item['Key'] for page in store.pages() for item in page.get('Contents',[])}
assert after==before
finally:
with db.pool.connection() as conn:
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON attachments').format(sql.Identifier(trigger)))
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
def test_compensation_preserves_committed_object(client):
from messenger.attachments import compensate
ah,_,_,chat=actors(client)
att=upload(client,ah,chat).json()
with client.app.state.db.pool.connection() as conn:
key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key']
# A retry after an unknown COMMIT result must never delete an already tracked object.
compensate(client.app.state.db,client.app.state.storage,key)
content=client.get('/api/v1/attachments/'+att['id']+'/content',headers=ah)
assert content.status_code==200 and content.content==image_bytes()
def test_stream_close_even_if_headers_cannot_be_sent(client):
import asyncio
import pytest
from messenger.attachments import ObjectResponse
ah,_,_,chat=actors(client)
att=upload(client,ah,chat).json()
with client.app.state.db.pool.connection() as conn:
key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key']
store=client.app.state.storage
opened=store.get(key)
response=ObjectResponse(opened,media_type='image/png')
async def fail_send(message):
raise OSError('synthetic disconnected transport')
async def receive():
return {'type':'http.disconnect'}
with pytest.raises(Exception):
asyncio.run(response({'type':'http','asgi':{'spec_version':'2.4'}},receive,fail_send))
assert opened.closed
acquired=0
try:
for _ in range(8):
assert store.gate.acquire(blocking=False)
acquired+=1
finally:
for _ in range(acquired):
store.gate.release()
def test_chunked_upload_is_limited_without_content_length(client):
ah,_,_,chat=actors(client)
chunks=iter([b'x'*1048576]*10+[b'x'])
response=client.post('/api/v1/chats/'+chat['id']+'/attachments?filename=chunked.png',
content=chunks,headers={**ah,'Content-Type':'image/png'})
assert response.status_code==413
assert response.json()['error']['code']=='payload_too_large'
+119
View File
@@ -0,0 +1,119 @@
"""Regressions for lab3 shared message results, real PostgreSQL (no mocks)."""
from concurrent.futures import ThreadPoolExecutor
import uuid
from psycopg import sql
from test_api import client, user, login
def setup_chat(c, extra=None):
u, body = user(c)
headers, _ = login(c, body)
members = [u['id']] + (extra or [])
chat = c.post('/api/v1/chats', headers=headers, json={'title':'Идемпотентность', 'member_ids':members}).json()
return u, headers, '/api/v1/chats/'+chat['id']+'/messages'
def test_instance_alias_on_success_and_errors(client):
responses = [client.get('/health/live'), client.get('/health/ready'), client.get('/api/v1/users/me'),
client.get('/unknown'), client.post('/api/v1/users', content='{', headers={'Content-Type':'application/json'})]
ids = [r.headers.get('X-Instance-Id') for r in responses]
assert ids[0] and len(set(ids)) == 1
assert all(r.headers.get('X-Request-Id') for r in responses)
def test_key_required_replay_and_conflict(client):
_, headers, path = setup_chat(client)
assert client.post(path, headers=headers, json={'text':'same'}).status_code == 400
for key in ['', 'a b', 'bad/', 'x'*129]:
assert client.post(path, headers={**headers,'Idempotency-Key':key},json={'text':'same'}).status_code == 400
h = {**headers, 'Idempotency-Key':'valid:._-123'}
first = client.post(path, headers=h, json={'text':'same'})
assert first.status_code == 201
replay = client.post(path, headers={**h,'Content-Type':'application/json'},content=' { "text" : "same" } ')
assert replay.status_code == 201 and replay.json() == first.json()
assert client.post(path, headers=h, json={'text':'other'}).status_code == 409
assert len(client.get(path,headers=headers).json()['items']) == 1
def test_concurrent_key_produces_one_message(client):
_, headers, path = setup_chat(client)
h = {**headers, 'Idempotency-Key':uuid.uuid4().hex}
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(lambda _: client.post(path,headers=h,json={'text':'concurrent'}), range(16)))
assert [r.status_code for r in results] == [201]*16
assert all(r.json() == results[0].json() for r in results)
assert client.get(path,headers=headers).json()['items'] == [results[0].json()]
def test_scope_and_replay_rechecks_authorization(client):
b, bb = user(client)
bh, _ = login(client, bb)
a, ah, path = setup_chat(client, [b['id']])
key = 'same-in-independent-scopes'
first = client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).json()
second = client.post(path,headers={**bh,'Idempotency-Key':key},json={'text':'b'})
assert second.status_code == 201 and second.json()['id'] != first['id']
chat = client.post('/api/v1/chats',headers=ah,json={'title':'Другая область','member_ids':[a['id']]}).json()
other_path = '/api/v1/chats/'+chat['id']+'/messages'
third = client.post(other_path,headers={**ah,'Idempotency-Key':key},json={'text':'a'})
assert third.status_code == 201 and third.json()['id'] != first['id']
outsider, ob = user(client)
oh, _ = login(client, ob)
assert client.post(path,headers={**oh,'Idempotency-Key':key},json={'text':'a'}).status_code == 404
assert client.delete('/api/v1/auth/sessions/current',headers=ah).status_code == 204
assert client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).status_code == 401
def test_expired_key_can_be_reused(client):
u, h, path = setup_chat(client)
key = uuid.uuid4().hex
first = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'first'}).json()
with client.app.state.db.pool.connection() as conn:
ttl = conn.execute('SELECT extract(epoch FROM expires_at-clock_timestamp()) AS seconds FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['seconds']
assert 86390 <= ttl <= 86400
conn.execute("UPDATE idempotency_records SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=%s AND key=%s", (u['id'],key))
second = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'second'})
assert second.status_code == 201 and second.json()['id'] != first['id']
assert len(client.get(path,headers=h).json()['items']) == 2
def test_rollback_side_effect_and_key_on_database_error(client):
u, h, path = setup_chat(client)
key = uuid.uuid4().hex
text = 'fault_' + uuid.uuid4().hex
function = 'fault_' + uuid.uuid4().hex
trigger = 'trigger_' + uuid.uuid4().hex
db = client.app.state.db
# AFTER INSERT fails after the side effect. This is a DB-only test fixture, not a production hook.
with db.pool.connection() as conn:
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN RAISE EXCEPTION 'synthetic insert fault'; END IF; RETURN NEW; END $$").format(sql.Identifier(function), sql.Literal(text)))
conn.execute(sql.SQL('CREATE TRIGGER {} AFTER INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
try:
failed = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
assert failed.status_code == 500
assert failed.headers['X-Instance-Id']
with db.pool.connection() as conn:
assert conn.execute('SELECT count(*) AS n FROM messages WHERE text=%s',(text,)).fetchone()['n'] == 0
assert conn.execute('SELECT count(*) AS n FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['n'] == 0
finally:
with db.pool.connection() as conn:
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(trigger)))
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
retried = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
assert retried.status_code == 201
assert client.get(path,headers=h).json()['items'] == [retried.json()]
def test_ready_requires_compatible_schema(client):
temporary = 'temporarily_unavailable_' + uuid.uuid4().hex
with client.app.state.db.pool.connection() as conn:
conn.execute(sql.SQL('ALTER TABLE idempotency_records RENAME TO {}').format(sql.Identifier(temporary)))
try:
r = client.get('/health/ready')
assert r.status_code == 503
assert r.json()['error']['code'] == 'unavailable'
assert client.get('/health/live').status_code == 200
finally:
with client.app.state.db.pool.connection() as conn:
conn.execute(sql.SQL('ALTER TABLE {} RENAME TO idempotency_records').format(sql.Identifier(temporary)))